fluent-plugin-jsonish 1.0.1 → 1.0.2

data.tar.gz.sig

@@ -1,2 +1 @@
-_�S�,�8�&�16�Q,b�x��$Ğ��%-KW����*�=��
-���U�cЖG�nz8���ǽ7U'�m1U.��4t�>뛠?
+���mH=�������e?�.�Lv]�8F�{��8c����[qt�l�	�>w��<}Wan�bA�V���,�D��9���j�x�(����T�)�"ԙ�jsk�ȉ�+H�so;�s%�s�}�������Jy�@s_6�fH����x>����ݼ�!,H�_�)��X�,�.�?:S�^���2�H�

data/README.md

@@ -1,4 +1,4 @@
-# jsonish, nginx_jsonish, and nodejs_bunyan parser plugins for Fluentd
+# jsonish, nginx_jsonish, nodejs_bunyan, and logstash parser plugins for Fluentd
 
 ## Overview
 The jsonish [parser plugin](http://docs.fluentd.org/articles/parser-plugin-overview) for fluentd.  It subclasses the JSONParser to allow for modifications to be made to input test before it is deserialized.  It subclasses the TimeParser to allow time format specifications using Ruby Time class method names instead of strftime format strings -- in particular, the iso8601 method.
@@ -29,6 +29,7 @@ gem install fluent-plugin-jsonish
   null_maps <how the patterns specified by null_pattern should be replaced>
   message_key <key to use for setting the 'message' key in the record>
   add_full_message <whether to the record 'full_message' key to the raw input>
+  move_keys <hash in JSON format, defaults to {}>
 </source>
 ```
 
@@ -39,6 +40,8 @@ gem install fluent-plugin-jsonish
 
 `maps`: an array containing "nulls", "slashes", or two-valued arrays containg (regex,replacement_text) pairs.  Defaults to [].
 
+`move_keys`: a hash specifying keys which should be moved after deseralization.  Each key should be the name of the key expected in the input JSON, and the corresponding value is the what the key needs to be in the record.  Use a null value to delete a key from the record.
+
 `null_pattern`: the pattern for how nulls are represented in the text.  Defaults to "-" (what nginx uses).
 
 `null_maps`: the an JSON array of two-valued arrays containing a sprintf regex string and substitution text (JSON: default [ [ \"(:\\s+)\"%s\"(\\s*[,}])\", \"\\1null\\2\" ], [ \"(:\\s+)%s(\\s*[,}])\", \"\\1null\\2\" ], [ \"(:\\s+)\\[\\s*%s\\s*\\](\\s*[,}])\", \"\\1[]\\2\" ], [ \"(:\\s+){\\s*%s\\*}(\\s*[,}])\", \"\\1{}\\2\" ] ])
@@ -97,3 +100,17 @@ The fluentd parser configuration for this input is straight-forward:
   [ standard parser or jsonish arguments ]
 </source>
 ```
+
+### logstash
+This is a parser for inputs whicare in [Logstash](https://gist.github.com/jordansissel/2996677) format.  Only two keys are required in this format ("@timestamp" and "@version").  The parser automatically maps "@timestamp" to time.  Both the "@version" and "@timestamp" keys are deleted, since they are part of the Logstash event meta data, not actual event data.
+
+The fluentd parser configuration for this input is straight-forward:
+
+```
+<source>
+  type tail
+  format logstash
+  path <application log file name>
+  [ standard parser or jsonish arguments ]
+</source>
+```

data/lib/fluent/plugin/parser_jsonish.rb

@@ -46,6 +46,7 @@ module Fluent
       ]
       config_param :message_key, :string, :default => nil
       config_param :add_full_message, :bool, :default => false
+      config_param :move_keys, :hash, :default => {}
 
       def initialize
         super
@@ -94,9 +95,6 @@ module Fluent
           end
         end
 
-        @message_key = conf['message_key']
-        @add_full_message = conf['add_full_message']
-
       end
 
       def parse(text)
@@ -116,6 +114,11 @@ module Fluent
             record['full_message'] = full_message
           end
 
+          @move_keys.map do |k,v|
+            record[v] = record.delete(k)
+          end
+          record.delete(nil)
+
           yield time, record
         end
 

data/lib/fluent/plugin/parser_logstash.rb

@@ -0,0 +1,16 @@
+require_relative 'parser_jsonish'
+
+module Fluent
+  class TextParser
+    class NodejsLogstash < JSONishParser
+      Plugin.register_parser('logstash', self)
+
+      def configure(conf)
+        super(conf)
+        @time_key = '@timestamp'
+        @move_keys.update({ '@version' => nil, '@timestamp' => nil })
+      end
+
+    end
+  end
+end

data/lib/fluent/plugin/parser_nginx_jsonish.rb

@@ -8,18 +8,18 @@ module Fluent
 
       def configure(conf)
 
-        if conf['maps'].is_a?(Array)
-          conf['maps'] = ([ 'slashes', 'nulls' ] + conf['maps']).uniq
+        if conf.key?('maps')
+          conf['maps'] = ([ 'slashes', 'nulls' ] + JSON.parse(conf['maps'])).uniq.to_json
         else
-          conf['maps'] = [ 'slashes', 'nulls' ]
-        end
-
-        if not (conf['message_key'] and conf['message_key'].is_empty?)
-          conf['message_key']  = 'request'
+          conf['maps'] = [ 'slashes', 'nulls' ].to_json
         end
 
         super(conf)
 
+        if @message_key.nil? or @message_key.empty?
+          @message_key = 'request'
+        end
+
       end
 
     end

data/lib/fluent/plugin/parser_nodejs_bunyan.rb

@@ -6,22 +6,16 @@ module Fluent
       Plugin.register_parser('nodejs_bunyan', self)
 
       def configure(conf)
-        super(conf.update({ 'time_key' => 'time', 'message_key' => 'msg'}))
+        super(conf)
+        @time_key = 'time'
+        @message_key = 'msg'
+        @move_keys.update({ 'v' => nil, 'msg' => nil, 'hostname' => 'host' })
       end
 
       def parse(text)
         super(text) do |time, record|
           # Map the developer-defined levels to  syslog levels.
           record['level'] = 8+(record['level']<=30?1:0)-(record['level']/10)
-          # Cannot be disabled in the library and serves no purpose,
-          # since it never varies.
-          record.delete('v')
-          # Fluent expects the host to be keyed with 'host', not 'hostname'.
-          record['host'] = record.delete('hostname')
-          record.delete('hostname')
-          # Passing 'message_key' only duplicates the key to 'message' --
-          # it does not delete the original key (which is unneeded).
-          record.delete(@message_key)
           yield time, record
         end
       end

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: fluent-plugin-jsonish
 version: !ruby/object:Gem::Version 
-  hash: 21
+  hash: 19
   prerelease: 
   segments: 
   - 1
   - 0
-  - 1
-  version: 1.0.1
+  - 2
+  version: 1.0.2
 platform: ruby
 authors: 
 - Alex Yamauchi
@@ -41,7 +41,7 @@ cert_chain:
   2Zk648Ep9HVPKmwoVuB75+xEQw==
   -----END CERTIFICATE-----
 
-date: 2017-06-16 00:00:00 Z
+date: 2017-08-17 00:00:00 Z
 dependencies: 
 - !ruby/object:Gem::Dependency 
   name: fluentd
@@ -76,6 +76,7 @@ files:
 - certs/oss@hotschedules.com.cert
 - fluent-plugin-jsonish.gemspec
 - lib/fluent/plugin/parser_jsonish.rb
+- lib/fluent/plugin/parser_logstash.rb
 - lib/fluent/plugin/parser_nginx_jsonish.rb
 - lib/fluent/plugin/parser_nodejs_bunyan.rb
 homepage: https://github.com/bodhi-space/fluent-plugin-jsonish