fluent-plugin-flowcounter-simple 0.0.4 → 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
- SHA1:
3
- metadata.gz: ea1531a6eca03db802d05acb32f7ac39cfd29576
4
- data.tar.gz: 6db938ffaa1c09dc074816b6491fc8dfa752b9f1
2
+ SHA256:
3
+ metadata.gz: 4f4d2e85abdbdf5c7b9e3bca8de7740066e6337262216bb09d7cd93e668c5977
4
+ data.tar.gz: beef12e960f130d3c1710d63e346c7524d410224e1548fc4c3bcc38643ed1ed8
5
5
  SHA512:
6
- metadata.gz: 3564af10781be5858eed85223f0657581a7140e729d994fba324bbb0222532d524408a1821182352316657f73f01182254daa62cad3e5de94257ace2b96e7939
7
- data.tar.gz: 7b327631a4d0cb1a082d26e0bebb3674549cf4b37184de7057993d942181d1e26d44c6481934d4272235758869383e192d71237cc582e860595535e5e8a0a8ce
6
+ metadata.gz: cb0cddb0135a5e1b8013d93490edcc872af9f2fe4d4f39206155704f00807a16a0b904e8bb3a5ad4049cdb8c5e81ecfc72bc8ffb581c6709abfcf4a1b64677fc
7
+ data.tar.gz: a1f1b6b1450ebae1c6269232924acb9769367dd13f31a4fa176ced494168f4aa6c42769f448247b15499195fbf3c33e5c6e509b88b67448e1bfa30c351c420f0
@@ -1,8 +1,11 @@
1
1
  rvm:
2
- - 1.9.3
3
- - 2.0.0
4
- - 2.1
5
- - 2.2
2
+ - 2.4.*
3
+ - 2.5.*
4
+ - 2.6.*
5
+ - 2.7.*
6
+
6
7
  gemfile:
7
8
  - Gemfile
8
- - Gemfile.v0.10
9
+
10
+ before_install:
11
+ - gem update bundler
@@ -1,3 +1,7 @@
1
+ ## 0.1.0 (2020/06/30)
2
+
3
+ * Use fluentd v1 API (repeatedly)
4
+
1
5
  ## 0.0.4 (2015/04/24)
2
6
 
3
7
  Enhancements:
data/README.md CHANGED
@@ -2,21 +2,15 @@
2
2
 
3
3
  Simple Fluentd Plugin to count number of messages and outputs to log
4
4
 
5
- ## Output Plugin Configuration
5
+ ## Requirements
6
6
 
7
- <match foo.bar.**>
8
- type flowcounter_simple
9
- unit second
10
- </match>
11
-
12
- This plugin does not emit, just writes counts into the log file as
13
-
14
- plugin:out_flowcounter_simple count:30 indicator:num unit:second
7
+ | fluent-plugin-flowcounter-simple | fluentd | ruby |
8
+ |-------------------|---------|------|
9
+ | >= 0.1.0 | >= v1.0 | >= 2.4 |
10
+ | < 0.0.4 | >= v0.12.0 | >= 2.1 |
15
11
 
16
12
  ## Filter Plugin Configuration
17
13
 
18
- Fluentd >= v0.12
19
-
20
14
  ```apache
21
15
  <filter foo.bar.**>
22
16
  type flowcounter_simple
@@ -28,6 +22,17 @@ This filter plugin pass through records, and writes counts into the log file as
28
22
 
29
23
  plugin:out_flowcounter_simple count:30 indicator:num unit:second
30
24
 
25
+ ## Output Plugin Configuration
26
+
27
+ <match foo.bar.**>
28
+ type flowcounter_simple
29
+ unit second
30
+ </match>
31
+
32
+ This plugin does not emit, just writes counts into the log file as
33
+
34
+ plugin:out_flowcounter_simple count:30 indicator:num unit:second
35
+
31
36
  ## Parameters
32
37
 
33
38
  - unit
@@ -1,7 +1,7 @@
1
1
  # -*- encoding: utf-8 -*-
2
2
  Gem::Specification.new do |gem|
3
3
  gem.name = "fluent-plugin-flowcounter-simple"
4
- gem.version = "0.0.4"
4
+ gem.version = "0.1.0"
5
5
  gem.authors = ["Naotoshi Seo"]
6
6
  gem.email = ["sonots@gmail.com"]
7
7
  gem.summary = %q{Simple Fluentd Plugin to count number of messages and outputs to log}
@@ -14,7 +14,7 @@ Gem::Specification.new do |gem|
14
14
  gem.test_files = gem.files.grep(%r{^(test|spec|features)/})
15
15
  gem.require_paths = ["lib"]
16
16
 
17
- gem.add_runtime_dependency "fluentd"
17
+ gem.add_runtime_dependency "fluentd", [">= 1.0"]
18
18
  gem.add_development_dependency "rake"
19
19
  gem.add_development_dependency "pry"
20
20
  gem.add_development_dependency "pry-nav"
@@ -1,21 +1,15 @@
1
- # I'm lazy to to implement filter plugin. Use output plugin instance
2
- require_relative 'out_flowcounter_simple'
3
- require 'forwardable'
1
+ require 'fluent/plugin/filter'
2
+ require_relative 'flowcounter_simple'
4
3
 
5
- class Fluent::FlowCounterSimpleFilter < Fluent::Filter
6
- Fluent::Plugin.register_filter('flowcounter_simple', self)
4
+ module Fluent::Plugin
5
+ class FlowCounterSimpleFilter < Filter
6
+ Fluent::Plugin.register_filter('flowcounter_simple', self)
7
7
 
8
- extend Forwardable
9
- attr_reader :output
10
- def_delegators :@output, :configure, :start, :shutdown, :flush_emit
8
+ include ::Fluent::FlowcounterSimple
11
9
 
12
- def initialize
13
- super
14
- @output = Fluent::FlowCounterSimpleOutput.new
10
+ def filter_stream(tag, es)
11
+ process_count(tag, es)
12
+ es
13
+ end
15
14
  end
16
-
17
- def filter_stream(tag, es)
18
- @output.emit(tag, es, Fluent::NullOutputChain.instance)
19
- es
20
- end
21
- end if defined?(Fluent::Filter)
15
+ end
@@ -0,0 +1,100 @@
1
+ require 'fluent/plugin_helper/thread'
2
+
3
+ module Fluent
4
+ module FlowcounterSimple
5
+ attr_accessor :last_checked
6
+
7
+ def self.included(klass)
8
+ klass.helpers :thread
9
+ klass.config_param :indicator, :string, :default => 'num'
10
+ klass.config_param :unit, :string, :default => 'second'
11
+ klass.config_param :comment, :string, :default => nil
12
+ end
13
+
14
+ def configure(conf)
15
+ super
16
+
17
+ @indicator_proc =
18
+ case @indicator
19
+ when 'num' then Proc.new { |es| es.size }
20
+ when 'byte' then Proc.new { |es|
21
+ count = 0
22
+ es.each { |time, record|
23
+ count += record.to_msgpack.size
24
+ }
25
+ count
26
+ }
27
+ else
28
+ raise Fluent::ConfigError, "flowcounter-simple count allows num/byte"
29
+ end
30
+ @unit =
31
+ case @unit
32
+ when 'second' then :second
33
+ when 'minute' then :minute
34
+ when 'hour' then :hour
35
+ when 'day' then :day
36
+ else
37
+ raise Fluent::ConfigError, "flowcounter-simple unit allows second/minute/hour/day"
38
+ end
39
+ @tick =
40
+ case @unit
41
+ when :second then 1
42
+ when :minute then 60
43
+ when :hour then 3600
44
+ when :day then 86400
45
+ else
46
+ raise Fluent::ConfigError, "@unit must be one of second/minute/hour/day"
47
+ end
48
+
49
+ @type_str = self.is_a?(Fluent::Plugin::Filter) ? 'filter' : 'out'
50
+ @output_proc =
51
+ if @comment
52
+ Proc.new { |count| "plugin:#{@type_str}_flowcounter_simple\tcount:#{count}\tindicator:#{@indicator}\tunit:#{@unit}\tcomment:#{@comment}" }
53
+ else
54
+ Proc.new { |count| "plugin:#{@type_str}_flowcounter_simple\tcount:#{count}\tindicator:#{@indicator}\tunit:#{@unit}" }
55
+ end
56
+
57
+ @count = 0
58
+ @mutex = Mutex.new
59
+ end
60
+
61
+ def start
62
+ super
63
+ thread_create(:flowcounter_simple_watch, &method(:watch))
64
+ end
65
+
66
+ def shutdown
67
+ super
68
+ end
69
+
70
+ def countup(count)
71
+ @mutex.synchronize {
72
+ @count = (@count || 0) + count
73
+ }
74
+ end
75
+
76
+ def flush_emit(step)
77
+ count, @count = @count, 0
78
+ if count > 0
79
+ log.info @output_proc.call(count)
80
+ end
81
+ end
82
+
83
+ def watch
84
+ # instance variable, and public accessable, for test
85
+ @last_checked = Fluent::EventTime.now
86
+ while thread_current_running?
87
+ sleep 0.1
88
+ if Fluent::EventTime.now - @last_checked >= @tick
89
+ now = Fluent::EventTime.now
90
+ flush_emit(now - @last_checked)
91
+ @last_checked = now
92
+ end
93
+ end
94
+ end
95
+
96
+ def process_count(tag, es)
97
+ countup(@indicator_proc.call(es))
98
+ end
99
+ end
100
+ end
@@ -1,106 +1,22 @@
1
- class Fluent::FlowCounterSimpleOutput < Fluent::Output
2
- Fluent::Plugin.register_output('flowcounter_simple', self)
1
+ require 'fluent/plugin/output'
2
+ require_relative 'flowcounter_simple'
3
3
 
4
- # To support log_level option implemented by Fluentd v0.10.43
5
- unless method_defined?(:log)
6
- define_method("log") { $log }
7
- end
8
-
9
- config_param :indicator, :string, :default => 'num'
10
- config_param :unit, :string, :default => 'second'
11
- config_param :comment, :string, :default => nil
12
-
13
- attr_accessor :last_checked
14
-
15
- def configure(conf)
16
- super
17
-
18
- @indicator_proc =
19
- case @indicator
20
- when 'num' then Proc.new {|record| 1 }
21
- when 'byte' then Proc.new {|record| record.to_msgpack.size }
22
- else
23
- raise Fluent::ConfigError, "flowcounter-simple count allows num/byte"
24
- end
25
- @unit =
26
- case @unit
27
- when 'second' then :second
28
- when 'minute' then :minute
29
- when 'hour' then :hour
30
- when 'day' then :day
31
- else
32
- raise Fluent::ConfigError, "flowcounter-simple unit allows second/minute/hour/day"
33
- end
34
- @tick =
35
- case @unit
36
- when :second then 1
37
- when :minute then 60
38
- when :hour then 3600
39
- when :day then 86400
40
- else
41
- raise RuntimeError, "@unit must be one of second/minute/hour/day"
42
- end
4
+ module Fluent::Plugin
5
+ class FlowCounterSimpleOutput < Output
6
+ Fluent::Plugin.register_output('flowcounter_simple', self)
43
7
 
44
- @output_proc =
45
- if @comment
46
- Proc.new {|count| "plugin:out_flowcounter_simple\tcount:#{count}\tindicator:#{@indicator}\tunit:#{@unit}\tcomment:#{@comment}" }
47
- else
48
- Proc.new {|count| "plugin:out_flowcounter_simple\tcount:#{count}\tindicator:#{@indicator}\tunit:#{@unit}" }
49
- end
8
+ include ::Fluent::FlowcounterSimple
50
9
 
51
- @count = 0
52
- @mutex = Mutex.new
53
- end
54
-
55
- def start
56
- super
57
- start_watch
58
- end
59
-
60
- def shutdown
61
- super
62
- @watcher.terminate
63
- @watcher.join
64
- end
65
-
66
- def countup(count)
67
- @mutex.synchronize {
68
- @count = (@count || 0) + count
69
- }
70
- end
71
-
72
- def flush_emit(step)
73
- count, @count = @count, 0
74
- if count > 0
75
- log.info @output_proc.call(count)
10
+ def prefer_buffered_processing
11
+ false
76
12
  end
77
- end
78
13
 
79
- def start_watch
80
- # for internal, or tests only
81
- @watcher = Thread.new(&method(:watch))
82
- end
83
-
84
- def watch
85
- # instance variable, and public accessable, for test
86
- @last_checked = Fluent::Engine.now
87
- while true
88
- sleep 0.1
89
- if Fluent::Engine.now - @last_checked >= @tick
90
- now = Fluent::Engine.now
91
- flush_emit(now - @last_checked)
92
- @last_checked = now
93
- end
14
+ def multi_workers_ready?
15
+ true
94
16
  end
95
- end
96
17
 
97
- def emit(tag, es, chain)
98
- count = 0
99
- es.each {|time,record|
100
- count += @indicator_proc.call(record)
101
- }
102
- countup(count)
103
-
104
- chain.next
18
+ def process(tag, es)
19
+ process_count(tag, es)
20
+ end
105
21
  end
106
22
  end
@@ -1,27 +1,28 @@
1
1
  require 'rubygems'
2
2
  require 'bundler'
3
- begin
4
- Bundler.setup(:default, :development)
5
- rescue Bundler::BundlerError => e
6
- $stderr.puts e.message
7
- $stderr.puts "Run `bundle install` to install missing gems"
8
- exit e.status_code
9
- end
3
+
10
4
  require 'test/unit'
5
+ require 'test/unit/rr'
11
6
 
12
7
  $LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
13
8
  $LOAD_PATH.unshift(File.dirname(__FILE__))
9
+
14
10
  require 'fluent/test'
15
- require 'fluent/plugin/out_flowcounter_simple'
16
- require 'fluent/plugin/filter_flowcounter_simple'
17
11
 
18
12
  class Test::Unit::TestCase
19
13
  def capture_log(log)
20
- tmp = log.out
21
- log.out = StringIO.new
22
- yield
23
- return log.out.string
24
- ensure
25
- log.out = tmp
14
+ if defined?(Fluent::Test::TestLogger) and log.is_a?(Fluent::Test::TestLogger) # v0.14
15
+ yield
16
+ log.out.logs.join("\n")
17
+ else
18
+ begin
19
+ tmp = log.out
20
+ log.out = StringIO.new
21
+ yield
22
+ return log.out.string
23
+ ensure
24
+ log.out = tmp
25
+ end
26
+ end
26
27
  end
27
28
  end
@@ -1,5 +1,6 @@
1
1
  require_relative '../helper'
2
- require "test/unit/rr"
2
+ require 'fluent/test/driver/filter'
3
+ require 'fluent/plugin/filter_flowcounter_simple'
3
4
 
4
5
  class FlowCounterSimpleFilterTest < Test::Unit::TestCase
5
6
  include Fluent
@@ -14,7 +15,7 @@ class FlowCounterSimpleFilterTest < Test::Unit::TestCase
14
15
  ]
15
16
 
16
17
  def create_driver(conf = CONFIG)
17
- Fluent::Test::FilterTestDriver.new(Fluent::FlowCounterSimpleFilter).configure(conf, true)
18
+ Fluent::Test::Driver::Filter.new(Fluent::Plugin::FlowCounterSimpleFilter).configure(conf)
18
19
  end
19
20
 
20
21
  def test_filter
@@ -26,24 +27,20 @@ class FlowCounterSimpleFilterTest < Test::Unit::TestCase
26
27
  d = create_driver
27
28
  filtered, out = filter(d, msgs)
28
29
  assert_equal msgs, filtered
29
- assert( out.include?("count:20"), out )
30
+ assert { out.include?("count:20") }
30
31
  end
31
32
 
32
33
  private
33
34
 
34
35
  def filter(d, msgs)
35
- stub(d.instance.output).start
36
- stub(d.instance.output).shutdown
37
- d.run {
36
+ d.run(default_tag: 'test') {
38
37
  msgs.each {|msg|
39
- d.filter(msg, @time)
38
+ d.feed(msg)
40
39
  }
41
40
  }
42
- out = capture_log(d.instance.output.log) do
41
+ out = capture_log(d.instance.log) do
43
42
  d.instance.flush_emit(0)
44
43
  end
45
- filtered = d.filtered_as_array
46
- filtered_msgs = filtered.map {|m| m[2] }
47
- [filtered_msgs, out]
44
+ [d.filtered_records, out]
48
45
  end
49
- end if defined?(Fluent::Filter)
46
+ end
@@ -1,4 +1,6 @@
1
1
  require_relative '../helper'
2
+ require 'fluent/test/driver/output'
3
+ require 'fluent/plugin/out_flowcounter_simple'
2
4
 
3
5
  class FlowCounterSimpleOutputTest < Test::Unit::TestCase
4
6
  def setup
@@ -10,60 +12,60 @@ class FlowCounterSimpleOutputTest < Test::Unit::TestCase
10
12
  ]
11
13
 
12
14
  def create_driver(conf=CONFIG,tag='test')
13
- Fluent::Test::OutputTestDriver.new(Fluent::FlowCounterSimpleOutput, tag).configure(conf)
15
+ Fluent::Test::Driver::Output.new(Fluent::Plugin::FlowCounterSimpleOutput).configure(conf)
14
16
  end
15
17
 
16
18
  def test_configure
17
19
  assert_nothing_raised {
18
- d = create_driver('')
20
+ create_driver('')
19
21
  }
20
22
  assert_nothing_raised {
21
- d = create_driver(CONFIG)
23
+ create_driver(CONFIG)
22
24
  }
23
25
  assert_nothing_raised {
24
- d = create_driver(CONFIG + %[indicator num])
26
+ create_driver(CONFIG + %[indicator num])
25
27
  }
26
28
  assert_nothing_raised {
27
- d = create_driver(CONFIG + %[indicator byte])
29
+ create_driver(CONFIG + %[indicator byte])
28
30
  }
29
31
  end
30
32
 
31
33
  def test_num
32
- d1 = create_driver(CONFIG, 'test.tag1')
33
- d1.run do
34
+ d1 = create_driver(CONFIG)
35
+ d1.run(default_tag: 'test.tag1') do
34
36
  10.times do
35
- d1.emit({'message'=> 'a' * 100})
36
- d1.emit({'message'=> 'b' * 100})
37
- d1.emit({'message'=> 'c' * 100})
37
+ d1.feed({'message'=> 'a' * 100})
38
+ d1.feed({'message'=> 'b' * 100})
39
+ d1.feed({'message'=> 'c' * 100})
38
40
  end
39
41
  end
40
42
  out = capture_log(d1.instance.log) { d1.instance.flush_emit(60) }
41
- assert( out.include?("count:30"), out )
43
+ assert { out.include?("count:30") }
42
44
  end
43
45
 
44
46
  def test_byte
45
- d1 = create_driver(CONFIG + %[indicator byte], 'test.tag1')
46
- d1.run do
47
+ d1 = create_driver(CONFIG + %[indicator byte])
48
+ d1.run(default_tag: 'test.tag1') do
47
49
  10.times do
48
- d1.emit({'message'=> 'a' * 100})
49
- d1.emit({'message'=> 'b' * 100})
50
- d1.emit({'message'=> 'c' * 100})
50
+ d1.feed({'message'=> 'a' * 100})
51
+ d1.feed({'message'=> 'b' * 100})
52
+ d1.feed({'message'=> 'c' * 100})
51
53
  end
52
54
  end
53
55
  out = capture_log(d1.instance.log) { d1.instance.flush_emit(60) }
54
- assert( out.include?("count:3360"), out )
56
+ assert { out =~ /count:\d+\tindicator:byte\tunit:second/ }
55
57
  end
56
58
 
57
59
  def test_comment
58
- d1 = create_driver(CONFIG + %[comment foobar], 'test.tag1')
59
- d1.run do
60
+ d1 = create_driver(CONFIG + %[comment foobar])
61
+ d1.run(default_tag: 'test.tag1') do
60
62
  1.times do
61
- d1.emit({'message'=> 'a' * 100})
62
- d1.emit({'message'=> 'b' * 100})
63
- d1.emit({'message'=> 'c' * 100})
63
+ d1.feed({'message'=> 'a' * 100})
64
+ d1.feed({'message'=> 'b' * 100})
65
+ d1.feed({'message'=> 'c' * 100})
64
66
  end
65
67
  end
66
68
  out = capture_log(d1.instance.log) { d1.instance.flush_emit(60) }
67
- assert( out.include?("comment:foobar"), out )
69
+ assert { out.include?("comment:foobar") }
68
70
  end
69
71
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: fluent-plugin-flowcounter-simple
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.0.4
4
+ version: 0.1.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Naotoshi Seo
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2015-04-23 00:00:00.000000000 Z
11
+ date: 2020-06-30 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: fluentd
@@ -16,14 +16,14 @@ dependencies:
16
16
  requirements:
17
17
  - - ">="
18
18
  - !ruby/object:Gem::Version
19
- version: '0'
19
+ version: '1.0'
20
20
  type: :runtime
21
21
  prerelease: false
22
22
  version_requirements: !ruby/object:Gem::Requirement
23
23
  requirements:
24
24
  - - ">="
25
25
  - !ruby/object:Gem::Version
26
- version: '0'
26
+ version: '1.0'
27
27
  - !ruby/object:Gem::Dependency
28
28
  name: rake
29
29
  requirement: !ruby/object:Gem::Requirement
@@ -105,7 +105,6 @@ files:
105
105
  - ".travis.yml"
106
106
  - CHANGELOG.md
107
107
  - Gemfile
108
- - Gemfile.v0.10
109
108
  - LICENSE.txt
110
109
  - README.md
111
110
  - Rakefile
@@ -113,6 +112,7 @@ files:
113
112
  - examples/output.conf
114
113
  - fluent-plugin-flowcounter-simple.gemspec
115
114
  - lib/fluent/plugin/filter_flowcounter_simple.rb
115
+ - lib/fluent/plugin/flowcounter_simple.rb
116
116
  - lib/fluent/plugin/out_flowcounter_simple.rb
117
117
  - test/helper.rb
118
118
  - test/plugin/test_filter_flowcounter_simple.rb
@@ -136,8 +136,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
136
136
  - !ruby/object:Gem::Version
137
137
  version: '0'
138
138
  requirements: []
139
- rubyforge_project:
140
- rubygems_version: 2.2.2
139
+ rubygems_version: 3.0.3
141
140
  signing_key:
142
141
  specification_version: 4
143
142
  summary: Simple Fluentd Plugin to count number of messages and outputs to log
@@ -1,4 +0,0 @@
1
- source "http://rubygems.org"
2
-
3
- gemspec
4
- gem 'fluentd', '~> 0.10.0'