fluent-plugin-path2tag 0.0.3 → 1.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 0e1d9b67029b9aa8766e87237ba322f53b0286f7
-  data.tar.gz: b7cbfd0f1750059b7fa22ce6d51a510b383a3033
+  metadata.gz: ae392e5bb11da4f51e7b6009341ad9212a6e1dda
+  data.tar.gz: b5d9ffd349a0f27c42d3cc6d7718d219d0c288de
 SHA512:
-  metadata.gz: ec6c901461ebf2b050e41fc223ced617845f16ffcb0188bed49cfeb1f672ae5d7dfd8437a980d051ebb9a332e1e350f2a96c716b24b87c1d636ad4d97ac955d6
-  data.tar.gz: 22d1ea01d89dfb5d44412e84cf5ed999e7868bd085c026607daa272eb9e8fc372c5ca83feeab43dd038e079e79732140230b5e9fd82d5b13bc7ef8383d96077e
+  metadata.gz: 9e19a37f51ab1d925e6bb09d2149e98c60abe750e56ab52638d2a9b5d4de2cd0cfa5f6e3d5031f98859bbbe22e001b8028d0e6a5bfbeccf5ad5c15bc28627fe8
+  data.tar.gz: 967be47a6ab16b03746463b28820d6d8582ed7581acf70161c071555cfe9e157477ed05ea6d60a8a96828754aa1252b6d81a08b3e525480382fd1083bb2c89fe

data/README.md

@@ -1,18 +1,19 @@
 # fluent-plugin-path2tag
 
 ## Overview
+A fluent output filter plugin for Nginx's logging.  
+Convert uri to tag. And reformat json data.
 
 ## Installation
-
 ```
 # for system installed fluentd
 $ gem install fluent-plugin-path2tag
 
 # for td-agent (Legacy)
-$ sudo /usr/lib64/fluent/ruby/bin/fluent-gem install ffluent-plugin-path2tag -v 0.0.1
+$ sudo /usr/lib64/fluent/ruby/bin/fluent-gem install ffluent-plugin-path2tag -v 1.0.0
 
 # for td-agent2 (with fluentd v0.12)
-$ sudo td-agent-gem install fluent-plugin-path2tag -v 0.0.1
+$ sudo td-agent-gem install fluent-plugin-path2tag -v 1.0.0
 ```
 
 ## Configuration
@@ -20,7 +21,6 @@ $ sudo td-agent-gem install fluent-plugin-path2tag -v 0.0.1
 ### Syntax
 ```
 path2tag <path_key_name> <data_key_name>
-
 ```
 
 ### Usage
@@ -48,4 +48,3 @@ path2tag <path_key_name> <data_key_name>
 Copyright (c) 2017- Shota Kuwahara  
 [MIT License](http://opensource.org/licenses/MIT)
 
-

data/lib/fluent/plugin/out_path2tag.rb

@@ -1,6 +1,11 @@
 class Fluent::Path2tagOutput < Fluent::Output
   Fluent::Plugin.register_output('path2tag', self)
 
+  # Define `router` method of v0.12 to support v0.10 or earlier
+  unless method_defined?(:router)
+    define_method("router") { Fluent::Engine }
+  end
+
   def configure(conf)
     super
 
@@ -8,49 +13,81 @@ class Fluent::Path2tagOutput < Fluent::Output
     conf.keys.select{|k| k =~ /^path2tag$/}.each do |key|
       path_key, data_key = conf[key].split(" ")
       if path_key.nil? || data_key.nil?
-        raise Fluent::ConfigError, "failed to parse path2tag at #{key} #{conf[key]}"
+        raise Fluent::ConfigError, "[path2tag] failed to parse path2tag at #{key} #{conf[key]}"
       end
 
       @rewriterules.push([path_key, data_key])
-      log.info "adding path2tag rule: #{key} #{@rewriterules.last}"
+      log.info "[path2tag] adding path2tag rule: #{key} #{@rewriterules.last}"
     end
 
     unless @rewriterules.length > 0
-      raise Fluent::ConfigError, "missing path2tag rule"
+      raise Fluent::ConfigError, "[path2tag] missing path2tag rule"
     end
   end
 
   def emit(tag, es, chain)
     es.each do |time,record|
-      rewrited_tag, newrecord = rewrite_tag_record(tag, record)
-      if newrecord.nil? then
-        router.emit(rewrited_tag, time, record)
+      rewrited_tag, newrecords = rewrite_tag_record(tag, record)
+      next if newrecords.nil? || tag == rewrited_tag
+      
+      log.info "[path2tag] change tag to #{rewrited_tag}"
+      
+      if newrecords.class == "Array"
+        newrecords.each do |r|
+          router.emit(rewrited_tag, time, r)
+        end
       else
-        router.emit(rewrited_tag, time, newrecord)
+        router.emit(rewrited_tag, time, newrecords)
       end
     end
 
     chain.next
   end
 
+  def parse(str = '')
+    array = str.scan(/(?:\\x..)+/).uniq.sort{ |a, b| b.length <=> a.length }
+    array.each do |reg|
+      s = [reg.gsub('\\x', 'x').split('x').slice(1..-1).join('')].pack('H*').force_encoding('utf-8')
+      str = str.gsub(reg, s)
+    end
+    return JSON.parse(str)
+  end
+
   def rewrite_tag_record(tag, record)
     path_key, data_key = @rewriterules[0]
-
+    
     unless record.has_key?(path_key)
-      log.warn "record has no path_key <#{path_key}>"
-      return "path2tag.clear", nil
+      log.warn "[path2tag] record has no path_key <#{path_key}>"
+      return tag, nil
     end
 
     unless record.has_key?(data_key)
-      log.warn "record has no data_key <#{data_key}>"
-      return "path2tag.clear", nil
+      log.warn "[path2tag] record has no data_key <#{data_key}>"
+      return tag, nil
+    end
+
+    if record[data_key].empty?
+      log.warn "[path2tag] record is empty <#{data_key}>"
+      return tag, nil
+    end
+
+    begin
+      newrecords = parse(record[data_key])
+    rescue => e
+      log.warn "[path2tag] JSON parse error!"
+      return tag, nil
     end
 
     newtag = record[path_key]
     if newtag.start_with?("/")
       newtag = newtag.sub("/", "")
     end
+
+    if newtag.end_with?("/")
+      newtag = newtag.chop
+    end
     newtag = newtag.gsub("/", ".")
-    return newtag, record[data_key]
+    
+    return newtag, newrecords
   end
 end

data/lib/fluent/plugin/path2tag/version.rb

@@ -1,7 +1,7 @@
 module Fluent
   module Plugin
     module Path2tag
-      VERSION = "0.0.3"
+      VERSION = "1.0.0"
     end
   end
 end

data/test/plugin/test_path2tag.rb

@@ -50,55 +50,106 @@ class Path2tagTest < Test::Unit::TestCase
 
   def test_emit_ok
     d1 = create_driver(CONFIG_OK, 'nginx.access')
+    body = '[{"fluentd_time":"2017-02-20 10:39:14 UTC","unique_id":"123ABC","device_token":"ABC123"}]'
     d1.run do
-      d1.emit({'request_uri' => '/logpose/endpoints/ios', 'request_body' => '{"fluentd_time":"2017-02-20 10:39:14 UTC","unique_id":"123ABC","device_token":"ABC123"}'})
+      d1.emit({'request_uri' => '/path/foo/bar', 'request_body' => body})
     end
     emits = d1.emits
     assert_equal 1, emits.length
-    assert_equal 'logpose.endpoints.ios', emits[0][0]
-    assert_equal '{"fluentd_time":"2017-02-20 10:39:14 UTC","unique_id":"123ABC","device_token":"ABC123"}', emits[0][2]
+    assert_equal 'path.foo.bar', emits[0][0]
+    assert_equal JSON.parse(body), emits[0][2]
   end
 
   def test_emit_no_path_key
     d1 = create_driver(CONFIG_NO_PATH_KEY, 'nginx.access')
+    body = '[{"fluentd_time":"2017-02-20 10:39:14 UTC","unique_id":"123ABC","device_token":"ABC123"}]'
     d1.run do
-      d1.emit({'request_uri' => '/logpose/endpoints/ios', 'request_body' => '{"fluentd_time":"2017-02-20 10:39:14 UTC","unique_id":"123ABC","device_token":"ABC123"}'})
+      d1.emit({'request_uri' => '/path/foo/bar', 'request_body' => body})
     end
     emits = d1.emits
-    assert_equal 1, emits.length
-    assert_equal 'path2tag.clear', emits[0][0]
+    assert_equal 0, emits.length
   end
 
   def test_emit_no_data_key
     d1 = create_driver(CONFIG_NO_DATA_KEY, 'nginx.access')
+    body = '[{"fluentd_time":"2017-02-20 10:39:14 UTC","unique_id":"123ABC","device_token":"ABC123"}]'
     d1.run do
-      d1.emit({'request_uri' => '/logpose/endpoints/ios', 'request_body' => '{"fluentd_time":"2017-02-20 10:39:14 UTC","unique_id":"123ABC","device_token":"ABC123"}'})
+      d1.emit({'request_uri' => '/path/foo/bar', 'request_body' => body})
     end
     emits = d1.emits
-    assert_equal 1, emits.length
-    assert_equal 'path2tag.clear', emits[0][0]
+    assert_equal 0, emits.length
   end
 
   def test_emit_path_format_1
     d1 = create_driver(CONFIG_OK, 'nginx.access')
+    body = '[{"fluentd_time":"2017-02-20 10:39:14 UTC","unique_id":"123ABC","device_token":"ABC123"}]'
     d1.run do
-       d1.emit({'request_uri' => '/logpose/endpoints/ios', 'request_body' => '{"fluentd_time":"2017-02-20 10:39:14 UTC","unique_id":"123ABC","device_token":"ABC123"}'})
+       d1.emit({'request_uri' => '/path/foo/bar', 'request_body' => body})
     end
     emits = d1.emits
     assert_equal 1, emits.length
-    assert_equal 'logpose.endpoints.ios', emits[0][0]
-    assert_equal '{"fluentd_time":"2017-02-20 10:39:14 UTC","unique_id":"123ABC","device_token":"ABC123"}', emits[0][2]
+    assert_equal 'path.foo.bar', emits[0][0]
+    assert_equal JSON.parse(body), emits[0][2]
   end
 
   def test_emit_path_format_2
     d1 = create_driver(CONFIG_OK, 'nginx.access')
+    body = '[{"fluentd_time":"2017-02-20 10:39:14 UTC","unique_id":"123ABC","device_token":"ABC123"}]'
     d1.run do
-       d1.emit({'request_uri' => 'logpose', 'request_body' => '{"fluentd_time":"2017-02-20 10:39:14 UTC","unique_id":"123ABC","device_token":"ABC123"}'})
+       d1.emit({'request_uri' => 'path/foo/bar', 'request_body' => body})
     end
     emits = d1.emits
     assert_equal 1, emits.length
-    assert_equal 'logpose', emits[0][0]
-    assert_equal '{"fluentd_time":"2017-02-20 10:39:14 UTC","unique_id":"123ABC","device_token":"ABC123"}', emits[0][2]
+    assert_equal 'path.foo.bar', emits[0][0]
+    assert_equal JSON.parse(body), emits[0][2]
   end
+
+  def test_emit_path_format_3
+    d1 = create_driver(CONFIG_OK, 'nginx.access')
+    body = '[{"fluentd_time":"2017-02-20 10:39:14 UTC","unique_id":"123ABC","device_token":"ABC123"}]'
+    d1.run do
+       d1.emit({'request_uri' => '/path/foo/bar/', 'request_body' => body})
+    end
+    emits = d1.emits
+    assert_equal 1, emits.length
+    assert_equal 'path.foo.bar', emits[0][0]
+    assert_equal JSON.parse(body), emits[0][2]
+  end
+
+  def test_emit_path_format_4
+    d1 = create_driver(CONFIG_OK, 'nginx.access')
+    body = '[{"fluentd_time":"2017-02-20 10:39:14 UTC","unique_id":"123ABC","device_token":"ABC123"}]'
+    d1.run do
+       d1.emit({'request_uri' => 'path', 'request_body' => body})
+    end
+    emits = d1.emits
+    assert_equal 1, emits.length
+    assert_equal 'path', emits[0][0]
+    assert_equal JSON.parse(body), emits[0][2]
+  end
+  
+  def test_emit_json_parse
+    d1 = create_driver(CONFIG_OK, 'nginx.access')
+    body = '{"fluentd_time":"2017-02-20 10:39:14 UTC","unique_id":"123ABC","device_token":"ABC123"}'
+    d1.run do
+       d1.emit({'request_uri' => 'path/foo/bar', 'request_body' => body})
+    end
+    emits = d1.emits
+    assert_equal 1, emits.length
+    assert_equal 'path.foo.bar', emits[0][0]
+    assert_equal JSON.parse(body), emits[0][2]
+  end
+  
+  def test_emit_json_parse_2
+    d1 = create_driver(CONFIG_OK, 'nginx.access')
+    body = '-'
+    d1.run do
+       d1.emit({'request_uri' => 'path/foo/bar', 'request_body' => body})
+    end
+    emits = d1.emits
+    assert_equal 0, emits.length
+  end
+
+
 end
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-path2tag
 version: !ruby/object:Gem::Version
-  version: 0.0.3
+  version: 1.0.0
 platform: ruby
 authors:
 - Shota Kuwahara
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-03-27 00:00:00.000000000 Z
+date: 2017-03-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: test-unit