fluent-plugin-multi-format-parser 0.1.1 → 1.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: b3ccd298b01b05ed96a4494b153bd4ee4abd2b8d
-  data.tar.gz: 5b8399121244fd4f5939a7acb8c2481c6450d1ce
+SHA256:
+  metadata.gz: 6c766b375521ddc5cc16503d3851d73ce3e7c995cd5f78dc3667dbe62966ff0b
+  data.tar.gz: 18f41d24351b487385d4433835cf0d8f81a648c86e3b99094e0127d08394a209
 SHA512:
-  metadata.gz: fefc45c5044df050487e7189d0071ee6021c29cac9ba4383e91273a08bdbf9cea851cd29ce160eac99df3c9994a775e48eb62f0c1db63bbc157aadccd2a14f3d
-  data.tar.gz: 8a69a5e16b154f21961b7227ee6e55a8ee7945825a966cc2e2badf5cde8ba9b921d5f671485947795979449c095d05d50089a4df401b826ec25aa1288b20c9d9
+  metadata.gz: cfbc21ac9ec34e10cbde396e3d34b4358121cc2430db5fd91f1b60576499531476862f23ed6858373841405cf742ee04bcd800d8218b5a85f9a0c77724361bd8
+  data.tar.gz: 14f2c6ecc7b3aed398f040d636797147ccd1c096e0573f8c3392d857972376905be0b1e46e67600d35e4c787f53c65ca7395495ce55da53fd6b2c0b7bb66fd71

data/.bundle/config

@@ -0,0 +1,2 @@
+---
+BUNDLE_PATH: "vendor/bundle"

data/README.md

@@ -2,6 +2,13 @@
 
 Parse format mixed logs.
 
+## Requirements
+
+| fluent-plugin-multi-format-parser | fluentd | ruby |
+|-------------------|---------|------|
+| >= 1.0.0 | >= v0.14.0 | >= 2.1 |
+|  < 1.0.0 | >= v0.12.0 | >= 1.9 |
+
 ## Installation
 
 Use RubyGems:
@@ -10,46 +17,108 @@ Use RubyGems:
 
 ## Configuration
 
-This plugin is a parser plugin. After installed, you can use `multi_format` in `format` supported plugins.
-Use multiple `<pattern>` to specify multiple format.
+This plugin is a parser plugin. After installed, you can use `multi_format` in `<parse>` supported plugins.
+Use multiple `<pattern>`s to specify multiple parser formats.
 
     <source>
       @type udp
       tag logs.multi
 
+      <parse>
+        @type multi_format
+        <pattern>
+          format apache
+        </pattern>
+        <pattern>
+          format json
+          time_key timestamp
+        </pattern>
+        <pattern>
+          format none
+        </pattern>
+      </parse>
+    </source>
+
+`multi_format` tries pattern matching from top to bottom and returns parsed result when matched.
+
+Available format patterns and parameters are depends on Fluentd parsers.
+See [parser plugin document](http://docs.fluentd.org/v1.0/articles/parser-plugin-overview) for more details.
+
+### For v1.0
+
+Put `<pattern>`s inside `<parse>`.
+
+    <filter app.**>
+      @type parser
+      key_name message
+      <parse>
+        @type multi_format
+        <pattern>
+          format json
+        </pattern>
+        <pattern>
+          format regexp
+          expression /...your regexp pattern.../
+        </pattern>
+        <pattern>
+          format none
+        </pattern>
+      </parse>
+    </filter>
+
+### For v0.12
+
+Use `format` instead of `<parse></parse>`.
+
+    <filter app.**>
+      @type parser
+      key_name message
+
       format multi_format
       <pattern>
-        format apache
+        format json
       </pattern>
       <pattern>
-        format json
-        time_key timestamp
+        format /...your regexp pattern.../
       </pattern>
       <pattern>
         format none
       </pattern>
-    </match>
+    </filter>
 
-`multi_format` tries pattern matching from top to bottom and returns parsed result when matched.
+### Adding format identity field
 
-Available format patterns and parameters are depends on Fluentd parsers.
-See [in_tail format document](http://docs.fluentd.org/articles/in_tail) for more details.
+Sometimes it may be useful to know which pattern was used. Since pareser usage
+may not support retagging, there is an option to add a format name field and/or
+index field.
 
-### For v0.14
+Example:
 
-This plugin handles `pattern` section manually, so v0.14's automatic parameter conversion doesn't work well.
-If you want to use this plugin with v0.14, you need to use v0.14 parser syntax like below
 
     <filter app.**>
       @type parser
       key_name message
-      <parse> # Use <parse> section for parser parameters
+      <parse>
         @type multi_format
+
+        # if set, add this key to record with value being pattern format name
+        # (format_name key)
+        format_key 'format'
+
         <pattern>
           format json
+          # set format name for this pattern. If unset, uses format name
+          # followed by index (in this case would be 'json#0')
+          format_name 'json'
+        </pattern>
+        <pattern>
+          format regexp
+          format_name 'MyRefex'
+          expression /...your regexp pattern.../
         </pattern>
         <pattern>
           format none
+          format_name 'unparsed'
         </pattern>
       </parse>
     </filter>
@@ -68,6 +137,6 @@ This plugin doesn't work with `multiline` parsers because parser itself doesn't
     <td>Copyright</td><td>Copyright (c) 2014- Masahiro Nakagawa</td>
   </tr>
   <tr>
-    <td>License</td><td>MIT License</td>
+    <td>License</td><td>Apache License (2.0)</td>
   </tr>
 </table>

data/Rakefile

@@ -1,6 +1,6 @@
 
 require 'bundler'
-Bundler::GemHelper.install_tasks
+require 'bundler/gem_tasks'
 
 require 'rake/testtask'
 
@@ -11,4 +11,3 @@ Rake::TestTask.new(:test) do |test|
 end
 
 task :default => [:build]
-

data/VERSION

@@ -1 +1 @@
-0.1.1
+1.1.0

data/fluent-plugin-multi-format-parser.gemspec

@@ -9,7 +9,7 @@ Gem::Specification.new do |gem|
   gem.version     = File.read("VERSION").strip
   gem.authors     = ["Masahiro Nakagawa"]
   gem.email       = "repeatedly@gmail.com"
-  gem.has_rdoc    = false
+  #gem.has_rdoc    = false
   #gem.platform    = Gem::Platform::RUBY
   gem.license     = 'Apache License (2.0)'
   gem.files       = `git ls-files`.split("\n")
@@ -17,6 +17,7 @@ Gem::Specification.new do |gem|
   gem.executables = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
   gem.require_paths = ['lib']
 
-  gem.add_dependency "fluentd", [">= 0.10.46", "< 2"]
+  gem.add_dependency "fluentd", [">= 0.14.0", "< 2"]
   gem.add_development_dependency "rake", ">= 0.9.2"
+  gem.add_development_dependency("test-unit", ["~> 3.3"])
 end

data/lib/fluent/plugin/parser_multi_format.rb

@@ -1,98 +1,49 @@
-require 'fluent/parser'
+require 'fluent/plugin/parser'
 
 module Fluent
-  class TextParser
-    if defined?(::Fluent::Parser)
-      class MultiFormatParser < Parser
-        Plugin.register_parser('multi_format', self)
+  module Plugin
+    class MultiFormatParser < Parser
+      Plugin.register_parser('multi_format', self)
+      config_param :format_key, :string, default: nil
 
-        def initialize
-          super
+      def initialize
+        super
 
-          @parsers = []
-        end
-
-        def configure(conf)
-          super
-
-          conf.elements.each { |e|
-            next unless ['pattern', 'format'].include?(e.name)
-            next if e['format'].nil? && (e['@type'] == 'multi_format')
-
-            parser = Plugin.new_parser(e['format'])
-            parser.configure(e)
-            @parsers << parser
-          }
-        end
-
-        def parse(text)
-          @parsers.each { |parser|
-            begin
-              parser.parse(text) { |time, record|
-                if time && record
-                  yield time, record
-                  return
-                end
-              }
-            rescue # ignore parser error
-            end
-          }
-
-          yield nil, nil
-        end
+        @parsers = []
+        @parser_format_names = []
       end
-    else # support old API. Will be removed.
-      class MultiFormatParser
-        include Configurable
-
-        def initialize
-          super
 
-          @parsers = []
-        end
-
-        def configure(conf)
-          super
-
-          conf.elements.each { |e|
-            next unless ['pattern', 'format'].include?(e.name)
-
-            parser = TextParser.new
-            parser.configure(e)
-            @parsers << parser.parser
-          }
-        end
+      def configure(conf)
+        super
+
+        conf.elements.each_with_index { |e, i|
+          next unless ['pattern', 'format'].include?(e.name)
+          next if e['format'].nil? && (e['@type'] == 'multi_format')
+          @parser_format_names << e.delete('format_name') || ""+e['format']+"#"+i.to_s
+          parser = Plugin.new_parser(e['format'])
+          parser.configure(e)
+          @parsers << parser
+        }
+      end
 
-        def parse(text)
-          @parsers.each { |parser|
-            begin
-              parser.call(text) { |time, record|
-                if time && record
-                  if block_given?
-                    yield time, record
-                    return
-                  else
-                    return time, record
-                  end
+      def parse(text)
+        @parsers.each_with_index { |parser, i|
+          begin
+            parser.parse(text) { |time, record|
+              if time && record
+                if @format_key
+                  record[@format_key] = @parser_format_names[i]
                 end
-              }
-            rescue # ignore parser error
-            end
-          }
-
-          if block_given?
-            yield nil, nil
-          else
-            return nil, nil
+                yield time, record
+                return
+              end
+            }
+          rescue # ignore parser error
           end
-        end
+        }
 
-        def call(*a, &b)
-          parse(*a, &b)
-        end
+        yield nil, nil
       end
-
-      register_template('multi_format', Proc.new { MultiFormatParser.new })
     end
   end
 end

data/test/test_parser_multi_format.rb

@@ -0,0 +1,61 @@
+require 'fluent/config'
+require 'fluent/test'
+require 'fluent/test/driver/parser'
+require 'fluent/plugin/parser'
+require 'fluent/plugin/parser_multi_format'
+require 'test/unit'
+
+class MultiFormatParserTest < ::Test::Unit::TestCase
+  def setup
+    Fluent::Test.setup
+  end
+
+  def create_driver(conf)
+    conf = Fluent::Config.parse(conf, "(test)", "(test_dir)", true)
+    Fluent::Test::Driver::Parser.new(Fluent::Plugin::MultiFormatParser.new).configure(conf)
+  end
+
+  def test_configure
+    conf = %[
+      @type multi_format
+      <pattern>
+        format apache
+      </pattern>
+      <pattern>
+        format json
+        time_key timestamp
+      </pattern>
+      <pattern>
+        format none
+      </pattern>
+    ]
+    d = create_driver(conf)
+    parsers = d.instance.instance_variable_get(:@parsers)
+    assert_instance_of(Fluent::Plugin::ApacheParser, parsers[0])
+    assert_instance_of(Fluent::Plugin::JSONParser, parsers[1])
+    assert_instance_of(Fluent::Plugin::NoneParser, parsers[2])
+  end
+
+  def test_parse
+    conf = %[
+      @type multi_format
+      <pattern>
+        format apache
+      </pattern>
+      <pattern>
+        format json
+        time_key timestamp
+      </pattern>
+      <pattern>
+        format none
+      </pattern>
+    ]
+    d = create_driver(conf)
+    d.instance.parse('{"k":"v"}') { |t, r|
+      assert_equal({"k" => "v"}, r)
+    }
+    d.instance.parse('hello') { |t, r|
+      assert_equal({"message" => "hello"}, r)
+    }
+  end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-multi-format-parser
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 1.1.0
 platform: ruby
 authors:
 - Masahiro Nakagawa
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2017-01-10 00:00:00.000000000 Z
+date: 2023-11-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -16,7 +16,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.10.46
+        version: 0.14.0
     - - "<"
       - !ruby/object:Gem::Version
         version: '2'
@@ -26,7 +26,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.10.46
+        version: 0.14.0
     - - "<"
       - !ruby/object:Gem::Version
         version: '2'
@@ -44,12 +44,27 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: 0.9.2
+- !ruby/object:Gem::Dependency
+  name: test-unit
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.3'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.3'
 description: Multi format parser plugin for Fluentd
 email: repeatedly@gmail.com
 executables: []
 extensions: []
 extra_rdoc_files: []
 files:
+- ".bundle/config"
 - ".travis.yml"
 - Gemfile
 - README.md
@@ -57,11 +72,12 @@ files:
 - VERSION
 - fluent-plugin-multi-format-parser.gemspec
 - lib/fluent/plugin/parser_multi_format.rb
+- test/test_parser_multi_format.rb
 homepage: https://github.com/repeatedly/fluent-plugin-multi-format-parser
 licenses:
 - Apache License (2.0)
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -76,9 +92,9 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.5.1
-signing_key: 
+rubygems_version: 3.4.1
+signing_key:
 specification_version: 4
 summary: Multi format parser plugin for Fluentd
-test_files: []
+test_files:
+- test/test_parser_multi_format.rb