fluent-plugin-json-parser 0.0.1 → 0.0.2

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in fluent-plugin-parser.gemspec
+gemspec

data/Rakefile

@@ -0,0 +1,11 @@
+#!/usr/bin/env rake
+require "bundler/gem_tasks"
+
+require 'rake/testtask'
+Rake::TestTask.new(:test) do |test|
+      test.libs << 'lib' << 'test'
+        test.pattern = 'test/**/test_*.rb'
+          test.verbose = true
+end
+
+task :default => :test

data/fluent-plugin-json-parser.gemspec

@@ -0,0 +1,19 @@
+# -*- encoding: utf-8 -*-
+Gem::Specification.new do |gem|
+  gem.name          = "fluent-plugin-json-parser"
+  gem.version       = "0.0.2"
+  gem.authors       = ["anarcher"]
+  gem.email         = ["anarcher@gmail.com"]
+  gem.description   = %q{fluentd plugin to json parse single field, or to combine log structure into single field}
+  gem.summary       = %q{plugin to parse/combine fluentd log messages}
+  gem.homepage      = "https://github.com/anarcher/fluent-plugin-json-parser"
+  gem.license       = "APLv2"
+
+  gem.files         = `git ls-files`.split($\)
+  gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
+  gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
+  gem.require_paths = ["lib"]
+
+  gem.add_development_dependency "rake"
+  gem.add_runtime_dependency "fluentd"
+end

data/lib/fluent/plugin/out_json_parser.rb

@@ -0,0 +1,51 @@
+require_relative './parser'
+
+class Fluent::ParserOutput < Fluent::Output
+    Fluent::Plugin.register_output('json_parser', self)
+    config_param :tag, :string, :default => nil
+    config_param :reserve_data, :bool, :default => false
+    config_param :key_name, :string
+
+    attr_reader :parser
+
+    def initialize
+        super
+        require 'time'
+    end
+
+    # Define `log` method for v0.10.42 or earlier
+    unless method_defined?(:log)
+        define_method("log") { $log }
+    end
+
+    def configure(conf)
+        super
+        if @key_name[0] == ":" 
+            @key_name = @key_name[1..-1].to_sym
+        end
+        @parser = FluentExt::JSONParser.new(log())
+    end
+
+    def emit(tag,es,chain)
+        tag = @tag || tag
+        es.each do |time,record|
+            raw_value = record[@key_name]
+            t,values = raw_value ? parse(raw_value) : [nil,nil]
+            t ||= time
+
+            r = @reserve_data ? record.merge(r) : values 
+
+            if r
+                Fluent::Engine.emit(tag,t,r)
+            end
+        end
+
+        chain.next
+    end
+
+    private
+
+    def parse(string)
+        return @parser.parse(string) 
+    end
+end

data/lib/fluent/plugin/parser.rb

@@ -0,0 +1,72 @@
+#
+# This module is copied from fluentd/lib/fluent/parser.rb and
+# fixed not to overwrite 'time' (reserve nil) when time not found in parsed string.
+
+module FluentExt
+  class GenericParser
+    include Fluent::Configurable
+
+    config_param :time_key, :string, :default => 'time'
+    config_param :time_format, :string, :default => nil
+    config_param :time_parse, :bool, :default => true
+
+    attr_accessor :log
+
+    def initialize(logger)
+      super()
+
+      @cache1_key = nil
+      @cache1_time = nil
+      @cache2_key = nil
+      @cache2_time = nil
+
+      @log = logger
+    end
+
+    def parse_time(record)
+      time = nil
+
+      unless @time_parse
+        return time, record
+      end
+
+      if value = record.delete(@time_key)
+        if @cache1_key == value
+          time = @cache1_time
+        elsif @cache2_key == value
+          time = @cache2_time
+        else
+          begin
+            time = if @time_format
+                     Time.strptime(value, @time_format).to_i
+                   else
+                     Time.parse(value).to_i
+                   end
+            @cache1_key = @cache2_key
+            @cache1_time = @cache2_time
+            @cache2_key = value
+            @cache2_time = time
+          rescue TypeError, ArgumentError => e
+            @log.warn "Failed to parse time", :key => @time_key, :value => value
+            record[@time_key] = value
+          end
+        end
+      end
+
+      return time, record
+    end
+  end
+
+  class JSONParser < GenericParser
+    def parse(text)
+      record = Yajl.load(text)
+      return parse_time(record)
+    rescue Yajl::ParseError
+      unless @suppress_parse_error_log
+        @log.warn "pattern not match(json): #{text.inspect}: #{$!}"
+      end
+      return nil, nil
+    end
+  end
+
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-json-parser
 version: !ruby/object:Gem::Version
-  version: 0.0.1
+  version: 0.0.2
   prerelease: 
 platform: ruby
 authors:
@@ -52,7 +52,12 @@ extensions: []
 extra_rdoc_files: []
 files:
 - .gitignore
+- Gemfile
 - LICENSE
+- Rakefile
+- fluent-plugin-json-parser.gemspec
+- lib/fluent/plugin/out_json_parser.rb
+- lib/fluent/plugin/parser.rb
 homepage: https://github.com/anarcher/fluent-plugin-json-parser
 licenses:
 - APLv2