fluent-plugin-groupcounter 0.2.0 → 0.2.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 86658cfcab0fe4d17fcbafd52f8baa4063905316
-  data.tar.gz: bd59e1fe0c665c9c942519d1eb026e9f258113d3
+  metadata.gz: c8cc4bf62fb23f54f31a2538cdf2ec1218e2970e
+  data.tar.gz: 8fe568adf912877846f3c26c0bb1426c2c0dba34
 SHA512:
-  metadata.gz: 92bab15c3c80a69a66c131de63a99d3cfcc3e4a61eea8618603413e02cd142cd93c897cd87571db89faa7270eb0e4318d1d4ac97237bc3dd824ecdd05bbddf16
-  data.tar.gz: 99cd15d0539ec3a0d898bdf4f6c8937111e523dca1acade1151c0e200500c1868717bd8196d76276991d671da9ec3f24f7bf977dea378d251e763d2589ea8c1e
+  metadata.gz: 13d484566258d0a0ce4b260baaebdfabb2763dcb5df908c684be4a8577173b6dbd2ddd590bbbf16ac170a080bb4b8a7675eefea15c72eed2152cd8c374b204e8
+  data.tar.gz: 93178c805d0fe9a19a206cfd3098a64cc51685f4daa0fb767a2bec45b25dc77b3f0656f85d075f2157b7b41ec63e83d0f88b05e17d1a0cd5d8e437b854e5461f

data/.gitignore

@@ -9,4 +9,5 @@ vendor
 doc/*
 tmp/*
 .yardoc
-
+pck/*
+.ruby-version

data/README.md

@@ -7,7 +7,7 @@ Fluentd plugin to count like SELECT COUNT(\*) GROUP BY.
 Assume inputs are coming as followings:
 
     apache.access: {"code":"200", "method":"GET", "path":"/index.html", "reqtime":"1.001" }
-    apache.access: {"code":"404", "method":"GET", "path":"/foo.html",   "reqtime":"2.002" }
+    apache.access: {"code":"202", "method":"GET", "path":"/foo.html",   "reqtime":"2.002" }
     apache.access: {"code":"200", "method":"GET", "path":"/index.html", "reqtime":"3.003" }
 
 Think of quering `SELECT COUNT(\*) GROUP BY code,method,path`. Configuration becomes as below:
@@ -22,7 +22,7 @@ Think of quering `SELECT COUNT(\*) GROUP BY code,method,path`. Configuration bec
 
 Output becomes like
 
-    groupcounter.apache.access: {"200_GET_/index.html_count":2, "404_GET_/foo.html_count":1}
+    groupcounter.apache.access: {"200_GET_/index.html_count":2, "202_GET_/foo.html_count":1}
 
 ## Parameters
 
@@ -34,6 +34,16 @@ Output becomes like
 
     Specify the delimiter to join `group_by_keys`. Default is '_'.
 
+* pattern\[1-20\]
+
+    Use `patternX` option to apply grouping more roughly. For example, adding a configuration for the above example as below
+
+         pattern1 2xx ^2\d\d
+
+    gives you an ouput like
+
+         groupcounter.apache.access: {"2xx_GET_/index.html_count":3}
+
 * group\_by\_expression (semi-required)
 
     Use an expression to group the event record. `group_by_keys` or `group_by_expression` is required.
@@ -89,7 +99,7 @@ Output becomes like
 
     gives you an output like
 
-        groupcounter.apache.access: {"200_GET_/index.html_reqtime_max":3.003, "404_GET_/foo.html_reqtime_max":2.002}
+        groupcounter.apache.access: {"200_GET_/index.html_reqtime_max":3.003, "202_GET_/foo.html_reqtime_max":2.002}
 
 * min\_key
 

data/fluent-plugin-groupcounter.gemspec

@@ -3,7 +3,7 @@ $:.push File.expand_path("../lib", __FILE__)
 
 Gem::Specification.new do |s|
   s.name        = "fluent-plugin-groupcounter"
-  s.version     = "0.2.0"
+  s.version     = "0.2.1"
   s.authors     = ["Ryosuke IWANAGA", "Naotoshi SEO"]
   s.email       = ["@riywo", "sonots@gmail.com"]
   s.homepage    = "https://github.com/riywo/fluent-plugin-groupcounter"

data/lib/fluent/plugin/out_groupcounter.rb

@@ -1,6 +1,8 @@
 class Fluent::GroupCounterOutput < Fluent::Output
   Fluent::Plugin.register_output('groupcounter', self)
 
+  PATTERN_MAX_NUM = 20
+
   def initialize
     super
     require 'pathname'
@@ -24,6 +26,7 @@ class Fluent::GroupCounterOutput < Fluent::Output
   config_param :min_suffix, :string, :default => '_min'
   config_param :avg_suffix, :string, :default => '_avg'
   config_param :store_file, :string, :default => nil
+  (1..PATTERN_MAX_NUM).each {|i| config_param "pattern#{i}".to_sym, :string, :default => nil }
 
   attr_accessor :count_interval
   attr_accessor :counts
@@ -69,6 +72,14 @@ class Fluent::GroupCounterOutput < Fluent::Output
 
     @group_by_keys = @group_by_keys.split(',') if @group_by_keys
 
+    @pattern = {}
+    (1..PATTERN_MAX_NUM).each do |i|
+      next unless conf["pattern#{i}"]
+      replace, regexp = conf["pattern#{i}"].split(/ +/, 2)
+      raise Fluent::ConfigError, "pattern#{i} does not contain 2 parameters" unless regexp
+      @pattern[replace] = Regexp.compile(regexp)
+    end
+
     if @store_file
       f = Pathname.new(@store_file)
       if (f.exist? && !f.writable_real?) || (!f.exist? && !f.parent.writable_real?)
@@ -231,6 +242,9 @@ class Fluent::GroupCounterOutput < Fluent::Output
       group_key = values.join(@delimiter)
     end
     group_key = group_key.to_s.force_encoding('ASCII-8BIT')
+
+    @pattern.each {|replace, regexp| break if group_key.gsub!(regexp, replace) }
+    group_key
   end
 
   def sum(a, b)

data/spec/out_groupcounter_spec.rb

@@ -347,6 +347,38 @@ describe Fluent::GroupCounterOutput do
       it { emit }
     end
 
+    context 'pattern' do
+      # NOTE: \\d should be just \d in config file
+      let(:config) { CONFIG + %[
+        group_by_expression ${method}_${path.split("?")[0].split("/")[2]}/${code}
+        delimiter _
+        pattern1 201 201$
+        pattern2 2xx 2\\d\\d$
+        pattern3 4xx 4\\d\\d$
+      ]}
+      let(:messages) do
+        [
+          {"code" => "200", "method" => "GET",  "path" => "/api/people/@me/@self?count=1", "reqtime" => 0.000 },
+          {"code" => "200", "method" => "POST", "path" => "/api/ngword?_method=check", "reqtime" => 1.001 },
+          {"code" => "400", "method" => "GET",  "path" => "/api/messages/@me/@outbox", "reqtime" => 2.002 },
+          {"code" => "201", "method" => "GET",  "path" => "/api/people/@me/@self", "reqtime" => 3.003 },
+        ]
+      end
+      let(:expected) do
+        {
+          "GET_people/201_count"=>1,
+          "GET_people/2xx_count"=>1,
+          "POST_ngword/2xx_count"=>1,
+          "GET_messages/4xx_count"=>1,
+        }
+      end
+      before do
+        Fluent::Engine.stub(:now).and_return(time)
+        Fluent::Engine.should_receive(:emit).with("count.#{tag}", time, expected)
+      end
+      it { emit }
+    end
+
   end
 end
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-groupcounter
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.2.1
 platform: ruby
 authors:
 - Ryosuke IWANAGA