fluent-plugin-multiline-parser 0.1.0

data/lib/fluent/plugin/out_deparser.rb

@@ -0,0 +1,94 @@
+class Fluent::DeparserOutput < Fluent::Output
+  Fluent::Plugin.register_output('deparser', self)
+
+  config_param :tag, :string, :default => nil
+  config_param :remove_prefix, :string, :default => nil
+  config_param :add_prefix, :string, :default => nil
+  config_param :format, :string
+  config_param :format_key_names, :string
+  config_param :key_name, :string, :default => 'message'
+  config_param :reserve_data, :bool, :default => false
+
+  def april_fool_emit(tag, es, chain)
+    es.each {|time,record|
+      keys = record.keys.shuffle
+      new_record = {@key_name => keys.map{|k| record[k]}.join(' ')}
+      router.emit(@tag, time, new_record)
+    }
+    chain.next
+  end
+
+  # Define `log` method for v0.10.42 or earlier
+  unless method_defined?(:log)
+    define_method("log") { $log }
+  end
+
+  def configure(conf)
+    if conf['tag'] == 'april.fool'
+      conf['format'] = '%s'
+      conf['format_key_names'] = 'x'
+    end
+
+    super
+
+    if @tag == 'april.fool'
+      m = method(:april_fool_emit)
+      (class << self; self; end).module_eval do
+        define_method(:emit, m)
+      end
+      return
+    end
+
+    if not @tag and not @remove_prefix and not @add_prefix
+      raise Fluent::ConfigError, "missing both of remove_prefix and add_prefix"
+    end
+    if @tag and (@remove_prefix or @add_prefix)
+      raise Fluent::ConfigError, "both of tag and remove_prefix/add_prefix must not be specified"
+    end
+    if @remove_prefix
+      @removed_prefix_string = @remove_prefix + '.'
+      @removed_length = @removed_prefix_string.length
+    end
+    if @add_prefix
+      @added_prefix_string = @add_prefix + '.'
+    end
+
+    @format_key_names = @format_key_names.split(',')
+    begin
+      dummy = @format % (["x"] * @format_key_names.length)
+    rescue ArgumentError
+      raise Fluent::ConfigError, "mismatch between placeholder of format and format_key_names"
+    end
+  end
+
+  def emit(tag, es, chain)
+    tag = if @tag
+            @tag
+          else
+            if @remove_prefix and
+                ( (tag.start_with?(@removed_prefix_string) and tag.length > @removed_length) or tag == @remove_prefix)
+              tag = tag[@removed_length..-1]
+            end
+            if @add_prefix
+              tag = if tag and tag.length > 0
+                      @added_prefix_string + tag
+                    else
+                      @add_prefix
+                    end
+            end
+            tag
+          end
+    if @reserve_data
+      es.each {|time,record|
+        record.update({@key_name => (@format % @format_key_names.map{|k| record[k]})})
+        router.emit(tag, time, record)
+      }
+    else
+      es.each {|time,record|
+        new_record = {@key_name => (@format % @format_key_names.map{|k| record[k]})}
+        router.emit(tag, time, new_record)
+      }
+    end
+    chain.next
+  end
+end

data/lib/fluent/plugin/out_parser.rb

@@ -0,0 +1,172 @@
+require 'fluent/parser'
+require 'thread_safe'
+
+class Fluent::ParserOutput < Fluent::Output
+
+  @@lines_buffer = ThreadSafe::Hash.new
+  Fluent::Plugin.register_output('parser', self)
+
+  config_param :tag, :string, :default => nil
+  config_param :remove_prefix, :string, :default => nil
+  config_param :add_prefix, :string, :default => nil
+  config_param :key_name, :string
+  config_param :reserve_data, :bool, :default => false
+  config_param :inject_key_prefix, :string, :default => nil
+  config_param :replace_invalid_sequence, :bool, :default => false
+  config_param :hash_value_field, :string, :default => nil
+  config_param :suppress_parse_error_log, :bool, :default => false
+  config_param :time_parse, :bool, :default => true
+
+  attr_reader :parser
+
+  def initialize
+    super
+    require 'time'
+  end
+
+  # Define `log` method for v0.10.42 or earlier
+  unless method_defined?(:log)
+    define_method("log") { $log }
+  end
+
+  def configure(conf)
+    super
+
+    if not @tag and not @remove_prefix and not @add_prefix
+      raise Fluent::ConfigError, "missing both of remove_prefix and add_prefix"
+    end
+    if @tag and (@remove_prefix or @add_prefix)
+      raise Fluent::ConfigError, "both of tag and remove_prefix/add_prefix must not be specified"
+    end
+    if @remove_prefix
+      @removed_prefix_string = @remove_prefix + '.'
+      @removed_length = @removed_prefix_string.length
+    end
+    if @add_prefix
+      @added_prefix_string = @add_prefix + '.'
+    end
+    @multiline_mode = conf['format'] =~ /multiline/
+    @receive_handler = if @multiline_mode
+                         method(:parse_multilines)
+                       else
+                         method(:parse_singleline)
+                       end
+
+    @parser = Fluent::Plugin.new_parser(conf['format'])
+    @parser.estimate_current_event = false
+    @parser.configure(conf)
+    if !@time_parse && @parser.parser.respond_to?("time_key=".to_sym)
+      # disable parse time
+      @parser.parser.time_key = nil
+    end
+
+    self
+  end
+  def parse_singleline(tag, time, record, line)
+    line.chomp!
+    @parser.parse(line) do |t,values|
+      if values
+        t ||= time
+        r = handle_parsed(tag, record, t, values)
+      else
+        log.warn "pattern not match with data #{tag} '#{line}'" unless @suppress_parse_error_log
+        if @reserve_data
+          t = time
+          r = handle_parsed(tag, record, time, {})
+        end
+      end
+    end
+  end
+
+  def parse_multilines(tag, time, record, line)
+    if @@lines_buffer.has_key?(tag)
+      matches = @parser.firstline?(line)
+      if matches
+        index = line.index(matches[0])
+        if index && index > 0
+            @@lines_buffer[tag] << line[0..index]
+            line = line[index..-1]
+        end
+        parse_singleline(tag, time, record, @@lines_buffer[tag])
+        @@lines_buffer[tag] = line
+      else
+        @@lines_buffer[tag] << line
+      end
+    else
+      @@lines_buffer[tag] = line
+    end
+  end
+
+
+  def emit(tag, es, chain)
+    tag = if @tag
+            @tag
+          else
+            if @remove_prefix and
+                ( (tag.start_with?(@removed_prefix_string) and tag.length > @removed_length) or tag == @remove_prefix)
+              tag = tag[@removed_length..-1]
+            end
+            if @add_prefix
+              tag = if tag and tag.length > 0
+                      @added_prefix_string + tag
+                    else
+                      @add_prefix
+                    end
+            end
+            tag
+          end
+    es.each do |time,record|
+      raw_value = record[@key_name]
+      begin
+        @receive_handler.call(tag, time, record, raw_value)
+      rescue Fluent::TextParser::ParserError => e
+        log.warn e.message unless @suppress_parse_error_log
+      rescue ArgumentError => e
+        if @replace_invalid_sequence
+          unless e.message.index("invalid byte sequence in") == 0
+            raise
+          end
+          replaced_string = replace_invalid_byte(raw_value)
+          @parser.parse(replaced_string) do |t,values|
+            if values
+              t ||= time
+              handle_parsed(tag, record, t, values)
+            else
+              log.warn "pattern not match with data '#{raw_value}'" unless @suppress_parse_error_log
+              if @reserve_data
+                t = time
+                handle_parsed(tag, record, time, {})
+              end
+            end
+          end
+        else
+          raise
+        end
+      rescue => e
+        log.warn "parse failed #{e.message}" unless @suppress_parse_error_log
+      end
+    end
+
+    chain.next
+  end
+
+  private
+
+  def handle_parsed(tag, record, t, values)
+    if values && @inject_key_prefix
+      values = Hash[values.map{|k,v| [ @inject_key_prefix + k, v ]}]
+    end
+    r = @hash_value_field ? {@hash_value_field => values} : values
+    if @reserve_data
+      r = r ? record.merge(r) : record
+    end
+    router.emit(tag, t, r)
+  end
+
+  def replace_invalid_byte(string)
+    replace_options = { invalid: :replace, undef: :replace, replace: '?' }
+    original_encoding = string.encoding
+    temporal_encoding = (original_encoding == Encoding::UTF_8 ? Encoding::UTF_16BE : Encoding::UTF_8)
+    string.encode(temporal_encoding, original_encoding, replace_options).encode(original_encoding)
+  end
+end

data/test/custom_parser.rb

@@ -0,0 +1,39 @@
+module Fluent
+  class TextParser
+    class KVPairParser < Parser
+      Plugin.register_parser('kv_pair', self)
+
+      # key<delim1>value is pair and <pair><delim2><pair> ...
+      # newline splits records
+      include Configurable
+
+      config_param :delim1, :string
+      config_param :delim2, :string
+
+      config_param :time_key, :string, :default => "time"
+      config_param :time_format, :string, :default => nil # time_format is configurable
+
+      def configure(conf)
+        super
+        @time_parser = TimeParser.new(@time_format)
+      end
+
+      def parse(text)
+        text.split("\n").each do |line|
+          pairs = text.split(@delim2)
+          record = {}
+          time = nil
+          pairs.each do |pair|
+            k, v = pair.split(@delim1, 2)
+            if k == @time_key
+              time = @time_parser.parse(v)
+            else
+              record[k] = v
+            end
+          end
+          yield time, record
+        end
+      end
+    end
+  end
+end

data/test/helper.rb

@@ -0,0 +1,31 @@
+require 'rubygems'
+require 'bundler'
+begin
+  Bundler.setup(:default, :development)
+rescue Bundler::BundlerError => e
+  $stderr.puts e.message
+  $stderr.puts "Run `bundle install` to install missing gems"
+  exit e.status_code
+end
+require 'test/unit'
+
+$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
+$LOAD_PATH.unshift(File.dirname(__FILE__))
+require 'fluent/test'
+unless ENV.has_key?('VERBOSE')
+  nulllogger = Object.new
+  nulllogger.instance_eval {|obj|
+    def method_missing(method, *args)
+      # pass
+    end
+  }
+  $log = nulllogger
+end
+
+require 'fluent/plugin/out_parser'
+require 'fluent/plugin/out_deparser'
+require 'fluent/plugin/filter_parser'
+require 'fluent/plugin/filter_deparser'
+
+class Test::Unit::TestCase
+end

data/test/plugin/test_deparser.rb

@@ -0,0 +1,149 @@
+require 'helper'
+
+class DeparserOutputTest < Test::Unit::TestCase
+  def setup
+    Fluent::Test.setup
+  end
+  
+  CONFIG = %[
+    remove_prefix test
+    add_prefix    combined
+    format        %s: %s %s %s
+    format_key_names host,path,status,size
+    key_name      fulltext
+    reserve_data  true
+  ]
+
+  def create_driver(conf=CONFIG,tag='test')
+    Fluent::Test::OutputTestDriver.new(Fluent::DeparserOutput, tag).configure(conf)
+  end
+
+  def test_configure
+    assert_nothing_raised {
+      d = create_driver %[
+        tag april.fool
+      ]
+    }
+
+    assert_raise(Fluent::ConfigError) {
+      d = create_driver('')
+    }
+    assert_raise(Fluent::ConfigError) {
+      d = create_driver %[
+        tag foo.bar
+      ]
+    }
+    assert_raise(Fluent::ConfigError) {
+      d = create_driver %[
+        format %s
+        format_key_names x
+      ]
+    }
+    assert_raise(Fluent::ConfigError) {
+      d = create_driver %[
+        tag foo.bar
+        remove_prefix foo
+        format %s
+        format_key_names x
+      ]
+    }
+    assert_raise(Fluent::ConfigError) {
+      d = create_driver %[
+        tag foo.bar
+        add_prefix foo
+        format %s
+        format_key_names x
+      ]
+    }
+    assert_raise(Fluent::ConfigError) {
+      d = create_driver %[
+        tag foo.bar
+        format_key_names x
+      ]
+    }
+    assert_raise(Fluent::ConfigError) {
+      d = create_driver %[
+        tag foo.bar
+        format %s
+      ]
+    }
+    assert_raise(Fluent::ConfigError) {
+      d = create_driver %[
+        tag foo.bar
+        format %s %s %s
+        format_key_names x,y
+      ]
+    }
+    assert_nothing_raised(Fluent::ConfigError) {
+      # mmm...
+      d = create_driver %[
+        tag foo.bar
+        format %s %s
+        format_key_names x,y,z
+      ]
+    }
+
+    d = create_driver %[
+      tag foo.bar
+      format %s: %s %s
+      format_key_names x,y,z
+    ]
+    assert_equal 'foo.bar', d.instance.tag
+    assert_equal '%s: %s %s', d.instance.format
+    assert_equal ['x','y','z'], d.instance.format_key_names
+    assert_equal 'message', d.instance.key_name
+    assert_equal false, d.instance.reserve_data
+  end
+
+  # CONFIG = %[
+  #   remove_prefix test
+  #   add_prefix    combined
+  #   format        %s: %s %s %s
+  #   format_key_names host path status size
+  #   key_name      fulltext
+  #   reserve_data  true
+  # ]
+  def test_emit
+    d1 = create_driver(CONFIG, 'test.in')
+    time = Time.parse("2012-01-02 13:14:15").to_i
+    d1.run do
+      d1.emit({'host'=>'xxx.local','path'=>'/f/1','status'=>'200','size'=>300}, time)
+      d1.emit({'host'=>'yyy.local','path'=>'/f/2','status'=>'302','size'=>512}, time)
+    end
+    emits = d1.emits
+    assert_equal 2, emits.length
+    first = emits[0]
+    assert_equal 'combined.in', first[0]
+    assert_equal time, first[1]
+    assert_equal 'xxx.local: /f/1 200 300', first[2]['fulltext']
+    assert_equal ['fulltext','host','path','size','status'], first[2].keys.sort
+    second = emits[1]
+    assert_equal 'combined.in', second[0]
+    assert_equal time, second[1]
+    assert_equal 'yyy.local: /f/2 302 512', second[2]['fulltext']
+    assert_equal ['fulltext','host','path','size','status'], second[2].keys.sort
+
+    d2 = create_driver(%[
+      tag combined
+      format %s: %s %s
+      format_key_names host,path,status
+    ], 'test.in')
+    time = Time.parse("2012-01-02 13:14:15").to_i
+    d2.run do
+      d2.emit({'host'=>'xxx.local','path'=>'/f/1','status'=>'200','size'=>300}, time)
+      d2.emit({'host'=>'yyy.local','path'=>'/f/2','status'=>'302','size'=>512}, time)
+    end
+    emits = d2.emits
+    assert_equal 2, emits.length
+    first = emits[0]
+    assert_equal 'combined', first[0]
+    assert_equal time, first[1]
+    assert_equal 'xxx.local: /f/1 200', first[2]['message']
+    assert_equal ['message'], first[2].keys.sort
+    second = emits[1]
+    assert_equal 'combined', second[0]
+    assert_equal time, second[1]
+    assert_equal 'yyy.local: /f/2 302', second[2]['message']
+    assert_equal ['message'], second[2].keys.sort
+  end
+end