fluent-plugin-multi-format-parser 1.0.0 → 1.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: da4c776b74c85a0624c5cc7f899b041ff8b0c196
-  data.tar.gz: 866aa7acf91781fdb2959c69ad8d94375f0f569c
+SHA256:
+  metadata.gz: 6c766b375521ddc5cc16503d3851d73ce3e7c995cd5f78dc3667dbe62966ff0b
+  data.tar.gz: 18f41d24351b487385d4433835cf0d8f81a648c86e3b99094e0127d08394a209
 SHA512:
-  metadata.gz: 86375df425d675d69613190c69ab05a8d0e304ce51e9456d2338b1ba9dd2c2070c33d9347b2f05c9ec245794f9e1937c8c0afe3bb1d29b7852637202f2918c4a
-  data.tar.gz: 87b2222ffa0188677498299b7570109f4cc11ac4a4a7e0a4c012ec4dccb82a6b54a33cedb0de2cdccbcdd44e7cf8d0b5f9f3cf6e163f2ebe2e3152c6d4704326
+  metadata.gz: cfbc21ac9ec34e10cbde396e3d34b4358121cc2430db5fd91f1b60576499531476862f23ed6858373841405cf742ee04bcd800d8218b5a85f9a0c77724361bd8
+  data.tar.gz: 14f2c6ecc7b3aed398f040d636797147ccd1c096e0573f8c3392d857972376905be0b1e46e67600d35e4c787f53c65ca7395495ce55da53fd6b2c0b7bb66fd71

data/.bundle/config

@@ -0,0 +1,2 @@
+---
+BUNDLE_PATH: "vendor/bundle"

data/README.md

@@ -44,7 +44,7 @@ Use multiple `<pattern>`s to specify multiple parser formats.
 Available format patterns and parameters are depends on Fluentd parsers.
 See [parser plugin document](http://docs.fluentd.org/v1.0/articles/parser-plugin-overview) for more details.
 
-### For v1.0/v0.14
+### For v1.0
 
 Put `<pattern>`s inside `<parse>`.
 
@@ -86,6 +86,43 @@ Use `format` instead of `<parse></parse>`.
       </pattern>
     </filter>
 
+### Adding format identity field
+
+Sometimes it may be useful to know which pattern was used. Since pareser usage
+may not support retagging, there is an option to add a format name field and/or
+index field.
+
+Example:
+
+
+    <filter app.**>
+      @type parser
+      key_name message
+      <parse>
+        @type multi_format
+
+        # if set, add this key to record with value being pattern format name
+        # (format_name key)
+        format_key 'format'
+
+        <pattern>
+          format json
+          # set format name for this pattern. If unset, uses format name
+          # followed by index (in this case would be 'json#0')
+          format_name 'json'
+        </pattern>
+        <pattern>
+          format regexp
+          format_name 'MyRefex'
+          expression /...your regexp pattern.../
+        </pattern>
+        <pattern>
+          format none
+          format_name 'unparsed'
+        </pattern>
+      </parse>
+    </filter>
+
 ### NOTE
 
 This plugin doesn't work with `multiline` parsers because parser itself doesn't store previous lines.
@@ -100,6 +137,6 @@ This plugin doesn't work with `multiline` parsers because parser itself doesn't
     <td>Copyright</td><td>Copyright (c) 2014- Masahiro Nakagawa</td>
   </tr>
   <tr>
-    <td>License</td><td>MIT License</td>
+    <td>License</td><td>Apache License (2.0)</td>
   </tr>
 </table>

data/Rakefile

@@ -1,6 +1,6 @@
 
 require 'bundler'
-Bundler::GemHelper.install_tasks
+require 'bundler/gem_tasks'
 
 require 'rake/testtask'
 
@@ -11,4 +11,3 @@ Rake::TestTask.new(:test) do |test|
 end
 
 task :default => [:build]
-

data/VERSION

@@ -1 +1 @@
-1.0.0
+1.1.0

data/fluent-plugin-multi-format-parser.gemspec

@@ -9,7 +9,7 @@ Gem::Specification.new do |gem|
   gem.version     = File.read("VERSION").strip
   gem.authors     = ["Masahiro Nakagawa"]
   gem.email       = "repeatedly@gmail.com"
-  gem.has_rdoc    = false
+  #gem.has_rdoc    = false
   #gem.platform    = Gem::Platform::RUBY
   gem.license     = 'Apache License (2.0)'
   gem.files       = `git ls-files`.split("\n")
@@ -19,4 +19,5 @@ Gem::Specification.new do |gem|
 
   gem.add_dependency "fluentd", [">= 0.14.0", "< 2"]
   gem.add_development_dependency "rake", ">= 0.9.2"
+  gem.add_development_dependency("test-unit", ["~> 3.3"])
 end

data/lib/fluent/plugin/parser_multi_format.rb

@@ -4,20 +4,22 @@ module Fluent
   module Plugin
     class MultiFormatParser < Parser
       Plugin.register_parser('multi_format', self)
+      config_param :format_key, :string, default: nil
 
       def initialize
         super
 
         @parsers = []
+        @parser_format_names = []
       end
 
       def configure(conf)
         super
 
-        conf.elements.each { |e|
+        conf.elements.each_with_index { |e, i|
           next unless ['pattern', 'format'].include?(e.name)
           next if e['format'].nil? && (e['@type'] == 'multi_format')
-
+          @parser_format_names << e.delete('format_name') || ""+e['format']+"#"+i.to_s
           parser = Plugin.new_parser(e['format'])
           parser.configure(e)
           @parsers << parser
@@ -25,10 +27,13 @@ module Fluent
       end
 
       def parse(text)
-        @parsers.each { |parser|
+        @parsers.each_with_index { |parser, i|
           begin
             parser.parse(text) { |time, record|
               if time && record
+                if @format_key
+                  record[@format_key] = @parser_format_names[i]
+                end
                 yield time, record
                 return
               end

data/test/test_parser_multi_format.rb

@@ -0,0 +1,61 @@
+require 'fluent/config'
+require 'fluent/test'
+require 'fluent/test/driver/parser'
+require 'fluent/plugin/parser'
+require 'fluent/plugin/parser_multi_format'
+require 'test/unit'
+
+class MultiFormatParserTest < ::Test::Unit::TestCase
+  def setup
+    Fluent::Test.setup
+  end
+
+  def create_driver(conf)
+    conf = Fluent::Config.parse(conf, "(test)", "(test_dir)", true)
+    Fluent::Test::Driver::Parser.new(Fluent::Plugin::MultiFormatParser.new).configure(conf)
+  end
+
+  def test_configure
+    conf = %[
+      @type multi_format
+      <pattern>
+        format apache
+      </pattern>
+      <pattern>
+        format json
+        time_key timestamp
+      </pattern>
+      <pattern>
+        format none
+      </pattern>
+    ]
+    d = create_driver(conf)
+    parsers = d.instance.instance_variable_get(:@parsers)
+    assert_instance_of(Fluent::Plugin::ApacheParser, parsers[0])
+    assert_instance_of(Fluent::Plugin::JSONParser, parsers[1])
+    assert_instance_of(Fluent::Plugin::NoneParser, parsers[2])
+  end
+
+  def test_parse
+    conf = %[
+      @type multi_format
+      <pattern>
+        format apache
+      </pattern>
+      <pattern>
+        format json
+        time_key timestamp
+      </pattern>
+      <pattern>
+        format none
+      </pattern>
+    ]
+    d = create_driver(conf)
+    d.instance.parse('{"k":"v"}') { |t, r|
+      assert_equal({"k" => "v"}, r)
+    }
+    d.instance.parse('hello') { |t, r|
+      assert_equal({"message" => "hello"}, r)
+    }
+  end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-multi-format-parser
 version: !ruby/object:Gem::Version
-  version: 1.0.0
+  version: 1.1.0
 platform: ruby
 authors:
 - Masahiro Nakagawa
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2017-12-14 00:00:00.000000000 Z
+date: 2023-11-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -44,12 +44,27 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: 0.9.2
+- !ruby/object:Gem::Dependency
+  name: test-unit
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.3'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.3'
 description: Multi format parser plugin for Fluentd
 email: repeatedly@gmail.com
 executables: []
 extensions: []
 extra_rdoc_files: []
 files:
+- ".bundle/config"
 - ".travis.yml"
 - Gemfile
 - README.md
@@ -57,11 +72,12 @@ files:
 - VERSION
 - fluent-plugin-multi-format-parser.gemspec
 - lib/fluent/plugin/parser_multi_format.rb
+- test/test_parser_multi_format.rb
 homepage: https://github.com/repeatedly/fluent-plugin-multi-format-parser
 licenses:
 - Apache License (2.0)
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -76,9 +92,9 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.6.13
-signing_key: 
+rubygems_version: 3.4.1
+signing_key:
 specification_version: 4
 summary: Multi format parser plugin for Fluentd
-test_files: []
+test_files:
+- test/test_parser_multi_format.rb