fluent-plugin-combiner 0.0.1 → 0.0.2

checksums.yaml

@@ -1,15 +1,15 @@
 ---
 !binary "U0hBMQ==":
   metadata.gz: !binary |-
-    MjI2MTFmNDQxOTgxNjNjZDg2ZTZmNGNiMzljYzhjNTZlNDU2MmExOQ==
+    OTQ1YTU4ODMzYTk2ZjQ4OTAyMjFhNWU4NTIwY2U2YmU2ODI5YzM0ZA==
   data.tar.gz: !binary |-
-    ZGYzZWU1NjExZjAyYTlhZTIyNzVlZWVmOWZkZjU1NmFhNDlkYWY3Mg==
+    NDc1M2M4ZDQ1M2M1M2I3OTZjMzBlMTg4MmIxY2ZiZjY2YzM5ODEyNA==
 SHA512:
   metadata.gz: !binary |-
-    ZGU5NjM4N2NhZjBmNTdiODFhYzhjZjgxZmRjY2E4YWY3YmE5YzNkMzVmYjc1
-    NjhjMjJkOWQ2MTc2Yzc1OTRjY2NkZDQxYzdhMWQzMmE0N2ZhMjIwNzdkNTRk
-    YjI0NzZkNDhhNDdhYTcwY2Q2MWMxMTIwYmE2ZDNmNGI1YjE3Mjk=
+    ZTVmYzZhMTkwZjgyZDYwYTliOGQwOGVjZTAwZjhkOGUyMGQ3YTU2MWMzMWU4
+    N2YwMDliYzhmNGU3NjZkMTc4NGUwODMyNjFhYmNkMzcxMDY2ZmExMDIzZTUy
+    NzVlNzA5ODllYjM5NzliNDkyYzQ0NjQyODcxYzM2OWQ2YWM5Nzg=
   data.tar.gz: !binary |-
-    MzM2ZjQ5ZWY5ZGViMjAxY2M0NGY3ZjM2ZWYwOGIzY2U1OTE1ZTk0ZTMyYmM1
-    MmVkNGQxMGQ5YjEzYThkNDBhOTBmNDRjMzMwOWM2MjBhOWQyODc4YTc2OTE5
-    YzQwNmY4MjUzY2E5OTI0MGJlNDYyMTdjNTU3ZGMwMGFhOWJkYTM=
+    ZGYxYWNjOTE5N2U3ZWU1MjY0M2QwYmYyZTk0NjVmNWQwNDJmZTZmNmQ0MDg4
+    MTI0NzNlNzA1MTA0MmY5NDI1OTIwY2JlNGUzMTE4NGRhODBkODMyNTNiODI2
+    ZjhlMDIwMDI5NTdiOWI0NmFhMTAzMDYwYTI0MGRjNjQ4OTk3Mjk=

data/README.md

@@ -1,5 +1,33 @@
 # fluent-plugin-combiner
 
+Fluentd plugin to count message keys, and make histogram.
+
+```
+$ echo '{"keys":["A",  "B",  "C",  "A"]}' | fluent-cat test.combine.input
+$ echo '{"keys":["A",  "B",  "D"]}' | fluent-cat test.combine.input
+```
+
+output is
+
+```
+2013-12-17 01:06:46 +0900 combined.input: {"hist":{"A":3, "B":2, "C":1, "D":1}, "sum":7, "len":4}
+```
+
+## Configuration
+
+```
+<match test.combine.**>
+    type combiner
+    count_key keys                      # input message tag to count
+    count_interval 5                    # count interval(second)
+    tag_prefix combined                 
+    input_tag_remove_prefix test.combine
+</match>
+```
+
+!!`tag` parameter overwrite `tag_prefix` and `input_tag_remove_prefix`
+ 
+
 ## Installation
 
 Add this line to your application's Gemfile:
@@ -14,9 +42,6 @@ Or install it yourself as:
 
     $ gem install fluent-plugin-combiner
 
-## Usage
-
-TODO: Write usage instructions here
 
 ## Contributing
 

data/bench/README.md

@@ -0,0 +1,30 @@
+Benchmark tool for Fluent event collector
+=========================================
+
+## Install
+
+    # genload.rb depends on fluent gem
+    $ gem install fluent
+
+## Usage
+
+    Usage: genload [options] <tag> <num>
+        -p, --port PORT                  fluent tcp port (default: 24224)
+        -h, --host HOST                  fluent host (default: 127.0.0.1)
+        -u, --unix                       use unix socket instead of tcp
+        -P, --path PATH                  unix socket path (default: /var/run/fluent/fluent.sock)
+        -r, --repeat NUM                 repeat number (default: 1)
+        -m, --multi NUM                  send multiple records at once (default: 1)
+        -c, --concurrent NUM             number of threads (default: 1)
+        -s, --size SIZE                  size of a record (default: 100)
+        -G, --no-packed                  don't use lazy deserialization optimize
+
+
+## Examples
+
+    # uses "benchmark.buffered" tag and sends 50,000 records
+    # -c: uses 10 threads/connections;
+    # -m: one message includes 20 record
+    # -r: repeats 100 times
+    ruby genload.rb benchamrk.buffered 50000 -c 10 -m 20 -r 100
+

data/bench/genload.rb

@@ -0,0 +1,152 @@
+require 'optparse'
+require 'fluent/env'
+
+op = OptionParser.new
+
+op.banner += " <tag> <num>"
+
+port = Fluent::DEFAULT_LISTEN_PORT
+host = '127.0.0.1'
+unix = false
+socket_path = Fluent::DEFAULT_SOCKET_PATH
+send_timeout = 20.0
+repeat = 1
+para = 1
+multi = 1
+record_len = 5
+packed = true
+
+config_path = Fluent::DEFAULT_CONFIG_PATH
+
+op.on('-p', '--port PORT', "fluent tcp port (default: #{port})", Integer) {|i|
+  port = s
+}
+
+op.on('-h', '--host HOST', "fluent host (default: #{host})") {|s|
+  host = s
+}
+
+op.on('-u', '--unix', "use unix socket instead of tcp", TrueClass) {|b|
+  unix = b
+}
+
+op.on('-P', '--path PATH', "unix socket path (default: #{socket_path})") {|s|
+  socket_path = s
+}
+
+op.on('-r', '--repeat NUM', "repeat number (default: 1)", Integer) {|i|
+  repeat = i
+}
+
+op.on('-m', '--multi NUM', "send multiple records at once (default: 1)", Integer) {|i|
+  multi = i
+}
+
+op.on('-l', '--record_len NUM', "a record to be send have NUM keys (default: 5)", Integer) {|i|
+  record_len = i
+}
+
+op.on('-c', '--concurrent NUM', "number of threads (default: 1)", Integer) {|i|
+  para = i
+}
+
+op.on('-G', '--no-packed', "don't use lazy deserialization optimize") {|i|
+  packed = false
+}
+
+(class<<self;self;end).module_eval do
+  define_method(:usage) do |msg|
+    puts op.to_s
+    puts "error: #{msg}" if msg
+    exit 1
+  end
+end
+
+begin
+  op.parse!(ARGV)
+
+  if ARGV.length != 2
+    usage nil
+  end
+
+  tag = ARGV.shift
+  num = ARGV.shift.to_i
+
+rescue
+  usage $!.to_s
+end
+
+require 'socket'
+require 'msgpack'
+require 'benchmark'
+
+def gen_word(len=nil)
+  len = rand(5) + 1 unless len
+  rand(36**len).to_s(36)
+end
+
+def gen_record(num=5, w_len=nil)
+  (1..num).reduce([]) {|ret| ret << gen_word(w_len)}
+end
+
+
+connector = Proc.new {
+  if unix
+    sock = UNIXSocket.open(socket_path)
+  else
+    sock = TCPSocket.new(host, port)
+  end
+
+  opt = [1, send_timeout.to_i].pack('I!I!')  # { int l_onoff; int l_linger; }
+  sock.setsockopt(Socket::SOL_SOCKET, Socket::SO_LINGER, opt)
+
+  opt = [send_timeout.to_i, 0].pack('L!L!')  # struct timeval
+  sock.setsockopt(Socket::SOL_SOCKET, Socket::SO_SNDTIMEO, opt)
+
+  sock
+}
+
+def gen_data(tag, multi=1, r_len=5)
+  time = Time.now.to_i
+  data = ''
+  multi.times do
+    record = {"keys"=>gen_record(r_len)}
+    [time, record].to_msgpack(data)
+  end
+  data = [tag, data].to_msgpack
+end
+
+size = 0 # sum of data.bytesize
+repeat.times do
+  puts "--- #{Time.now}"
+  Benchmark.bm do |x|
+    start = Time.now
+
+    lo = num / para / multi
+    lo = 1 if lo == 0
+
+    x.report do
+      (1..para).map {
+        Thread.new do
+          sock = connector.call
+          lo.times do
+            data = gen_data(tag, multi, record_len)
+            size += data.bytesize
+            sock.write data
+          end
+          sock.close
+        end
+      }.each {|t|
+        t.join
+      }
+    end
+
+    finish = Time.now
+    elapsed = finish - start
+
+    puts "% 10.3f Mbps" % [size*lo*para/elapsed/1000/1000]
+    puts "% 10.3f records/sec" % [lo*para*multi/elapsed]
+  end
+
+end
+

data/fluent-plugin-combiner.gemspec

@@ -4,7 +4,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |spec|
   spec.name          = "fluent-plugin-combiner"
-  spec.version       = "0.0.1"
+  spec.version       = "0.0.2"
   spec.authors       = ["karahiyo"]
   spec.email         = ["a.ryuklnm@gmail.com"]
   spec.summary       = "Combine buffer output data to cut-down net-i/o load"

data/lib/fluent/plugin/out_combiner.rb

@@ -3,8 +3,8 @@ module Fluent
     Fluent::Plugin.register_output('combiner', self)
 
     # config_param :hoge, :string, :default => 'hoge'
-    config_param :tag, :string, :default => 'combined'
-    config_param :tag_prefix, :string, :default => nil
+    config_param :tag, :string, :default => nil
+    config_param :tag_prefix, :string, :default => 'combined'
     config_param :input_tag_remove_prefix, :string, :default => nil
     config_param :count_interval, :time, :default => 60
     config_param :count_key, :string, :default => 'keys'
@@ -21,7 +21,8 @@ module Fluent
       super
 
       @tick = @count_interval.to_i if @count_interval
-      @tag_prefix_string = @tag_prefix ? @tag_prefix + '.' : @tag + '.'
+      @tag_str = @tag + '.' if @tag
+      @tag_prefix_string = @tag_prefix + '.' if @tag_prefix
       if @input_tag_remove_prefix
         @remove_prefix_string = @input_tag_remove_prefix + '.' 
         @remove_prefix_length = @remove_prefix_string.length
@@ -49,13 +50,20 @@ module Fluent
 
     def flush_emit
       flushed = flush
-      Fluent::Engine.emit(@tag,  Fluent::Engine.now,  flushed)
+      now = Fluent::Engine.now
+      flushed.each do |tag, message|
+        Fluent::Engine.emit(tag, now, message)
+      end
     end
 
     def generate_output(data)
       output = {}
       data.each do |tag, hist|
-        output[add_prefix(stripped_tag(tag))] = hist
+        if @tag
+          output[@tag] = hist
+        else
+          output[add_prefix(stripped_tag(tag))] = hist
+        end
       end
       output
     end

data/test/plugin/test_out_combiner.rb

@@ -19,7 +19,7 @@ class CombinerOutputTest < Test::Unit::TestCase
   def test_configure
     d = create_driver
     assert_equal 5, d.instance.tick
-    assert_equal 'combined', d.instance.tag
+    assert_equal nil, d.instance.tag
     assert_equal 'keys', d.instance.count_key
     assert_equal 'combined', d.instance.tag_prefix
     assert_equal 'test.input', d.instance.input_tag_remove_prefix

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-combiner
 version: !ruby/object:Gem::Version
-  version: 0.0.1
+  version: 0.0.2
 platform: ruby
 authors:
 - karahiyo
@@ -64,6 +64,8 @@ files:
 - LICENSE.txt
 - README.md
 - Rakefile
+- bench/README.md
+- bench/genload.rb
 - fluent-plugin-combiner.gemspec
 - lib/fluent/plugin/out_combiner.rb
 - test/helper.rb