fluent-multi-format-parser 1.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: c8a91fa22b9e9593dc63a93f6022c453b4c3c2070b6f993ecbfa3077ead19820
+  data.tar.gz: 490c2f7916f5b9eda4d5385c30131ef738933e134571756f2f8bd85576487f4e
+SHA512:
+  metadata.gz: 5fde9c3e50455ed13de3597f99d5cf6195055031d34403e222ce8c49161a6b2e14f1f610274f8f66321db03d9371464fdba7ffc63a0864bde4a80993bcdcb4a7
+  data.tar.gz: d18a37810a9272b5b963bf7ebd8baab1f39f8416f54ffb78f1fd8ed60c21a84f8be2dd6ffcb1761266be16f1a202cc458ccf86d8a65d02110479e50d8577539e

data/.travis.yml

@@ -0,0 +1,17 @@
+language: ruby
+
+rvm:
+  - 2.1
+  - 2.2.4
+  - 2.3.1
+  - ruby-head
+
+branches:
+  only:
+    - master
+
+matrix:
+  allow_failures:
+    - rvm: ruby-head
+
+script: bundle exec rake test

data/Gemfile

@@ -0,0 +1,3 @@
+source "https://rubygems.org"
+
+gemspec

data/README.md

@@ -0,0 +1,145 @@
+# Multi format parser plugin for Fluentd
+
+Parse format mixed logs.
+
+## Requirements
+
+| fluent-plugin-multi-format-parser | fluentd | ruby |
+|-------------------|---------|------|
+| >= 1.0.0 | >= v0.14.0 | >= 2.1 |
+|  < 1.0.0 | >= v0.12.0 | >= 1.9 |
+
+## Installation
+
+Use RubyGems:
+
+    fluent-gem install fluent-plugin-multi-format-parser
+
+## Configuration
+
+This plugin is a parser plugin. After installed, you can use `multi_format` in `<parse>` supported plugins.
+Use multiple `<pattern>`s to specify multiple parser formats.
+
+    <source>
+      @type udp
+      tag logs.multi
+
+      <parse>
+        @type multi_format
+        <pattern>
+          format apache
+        </pattern>
+        <pattern>
+          format json
+          time_key timestamp
+        </pattern>
+        <pattern>
+          format none
+        </pattern>
+      </parse>
+    </source>
+
+`multi_format` tries pattern matching from top to bottom and returns parsed result when matched.
+
+Available format patterns and parameters are depends on Fluentd parsers.
+See [parser plugin document](http://docs.fluentd.org/v1.0/articles/parser-plugin-overview) for more details.
+
+### For v1.0
+
+Put `<pattern>`s inside `<parse>`.
+
+    <filter app.**>
+      @type parser
+      key_name message
+      <parse>
+        @type multi_format
+        <pattern>
+          format json
+        </pattern>
+        <pattern>
+          format regexp
+          expression /...your regexp pattern.../
+        </pattern>
+        <pattern>
+          format none
+        </pattern>
+      </parse>
+    </filter>
+
+### For v0.12
+
+Use `format` instead of `<parse></parse>`.
+
+    <filter app.**>
+      @type parser
+      key_name message
+
+      format multi_format
+      <pattern>
+        format json
+      </pattern>
+      <pattern>
+        format /...your regexp pattern.../
+      </pattern>
+      <pattern>
+        format none
+      </pattern>
+    </filter>
+
+### Adding format identity field
+
+Sometimes it may be useful to know which pattern was used. Since pareser usage
+may not support retagging, there is an option to add a format name field and/or
+index field.
+
+Example:
+
+
+    <filter app.**>
+      @type parser
+      key_name message
+      <parse>
+        @type multi_format
+
+        # if set, add this key to record with value being pattern format name
+        # (format_name key)
+        format_key 'format'
+
+        # If set, add index of the matching pattern as this key (0-based)
+        index_key 'parser_index'
+
+        <pattern>
+          format json
+          # set format name for this pattern. If unset, uses format name
+          # followed by index (in this case would be 'json#0')
+          format_name 'json'
+        </pattern>
+        <pattern>
+          format regexp
+          format_name 'MyRefex'
+          expression /...your regexp pattern.../
+        </pattern>
+        <pattern>
+          format none
+          format_name 'unparsed'
+        </pattern>
+      </parse>
+    </filter>
+
+### NOTE
+
+This plugin doesn't work with `multiline` parsers because parser itself doesn't store previous lines.
+
+## Copyright
+
+<table>
+  <tr>
+    <td>Author</td><td>Masahiro Nakagawa <repeatedly@gmail.com></td>
+  </tr>
+  <tr>
+    <td>Copyright</td><td>Copyright (c) 2014- Masahiro Nakagawa</td>
+  </tr>
+  <tr>
+    <td>License</td><td>Apache License (2.0)</td>
+  </tr>
+</table>

data/Rakefile

@@ -0,0 +1,14 @@
+
+require 'bundler'
+Bundler::GemHelper.install_tasks
+
+require 'rake/testtask'
+
+Rake::TestTask.new(:test) do |test|
+  test.libs << 'lib' << 'test'
+  test.test_files = FileList['test/*.rb']
+  test.verbose = true
+end
+
+task :default => [:build]
+

data/VERSION

@@ -0,0 +1 @@
+1.1.0

data/fluent-plugin-multi-format-parser.gemspec

@@ -0,0 +1,22 @@
+# encoding: utf-8
+$:.push File.expand_path('../lib', __FILE__)
+
+Gem::Specification.new do |gem|
+  gem.name        = "fluent-multi-format-parser"
+  gem.description = "Multi format parser plugin for Fluentd"
+  gem.homepage    = "https://github.com/masterashu-motorq/fluentd-multi-parser"
+  gem.summary     = gem.description
+  gem.version     = File.read("VERSION").strip
+  gem.authors     = ["Ashutosh Chauhan"]
+  gem.email       = "ashutosh@motorq.com"
+  #gem.has_rdoc    = false
+  #gem.platform    = Gem::Platform::RUBY
+  gem.license     = 'Apache-2.0'
+  gem.files       = `git ls-files`.split("\n")
+  gem.test_files  = `git ls-files -- {test,spec,features}/*`.split("\n")
+  gem.executables = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  gem.require_paths = ['lib']
+
+  gem.add_dependency "fluentd", [">= 0.14.0", "< 2"]
+  gem.add_development_dependency "rake", ">= 0.9.2"
+end

data/lib/fluent/plugin/parser_multi_format.rb

@@ -0,0 +1,60 @@
+require 'fluent/plugin/parser'
+
+SubParser = Struct.new(:parser, :index, :name)
+FORMAT_NAME = "format_name"
+
+module Fluent
+  module Plugin
+    class MultiFormatParser < Parser
+      Plugin.register_parser('multi_format', self)
+      config_param :format_key, :string, default: ''
+      config_param :index_key, :string, default: ''
+
+      def initialize
+        super
+
+        @parsers = []
+        @format_key = ''
+        @index_key = ''
+      end
+
+      def configure(conf)
+        super
+
+        conf.elements.each_with_index { |e, i|
+          next unless ['pattern', 'format'].include?(e.name)
+          next if e['format'].nil? && (e['@type'] == 'multi_format')
+          parser = SubParser.new
+          format_name = e.delete(FORMAT_NAME) || ""+e['format']+"#"+i.to_s
+          parser.parser = Plugin.new_parser(e['format'])
+          parser.index = i
+          parser.name = format_name
+          parser.parser.configure(e)
+          @parsers << parser
+        }
+      end
+
+      def parse(text)
+        @parsers.each { |subparser|
+          begin
+            subparser.parser.parse(text) { |time, record|
+              if time && record
+                if ! (@format_key.nil? || @format_key.empty?)
+                  record[@format_key] = subparser.name
+                end
+                if ! (@index_key.nil? || @index_key.empty?)
+                  record[@index_key] = subparser.index
+                end
+                yield time, record
+                return
+              end
+            }
+          rescue # ignore parser error
+          end
+        }
+
+        yield nil, nil
+      end
+    end
+  end
+end

metadata

@@ -0,0 +1,83 @@
+--- !ruby/object:Gem::Specification
+name: fluent-multi-format-parser
+version: !ruby/object:Gem::Version
+  version: 1.1.0
+platform: ruby
+authors:
+- Ashutosh Chauhan
+autorequire:
+bindir: bin
+cert_chain: []
+date: 2022-09-02 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: fluentd
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.14.0
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '2'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.14.0
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '2'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.9.2
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.9.2
+description: Multi format parser plugin for Fluentd
+email: ashutosh@motorq.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".travis.yml"
+- Gemfile
+- README.md
+- Rakefile
+- VERSION
+- fluent-plugin-multi-format-parser.gemspec
+- lib/fluent/plugin/parser_multi_format.rb
+homepage: https://github.com/masterashu-motorq/fluentd-multi-parser
+licenses:
+- Apache-2.0
+metadata: {}
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.3.7
+signing_key:
+specification_version: 4
+summary: Multi format parser plugin for Fluentd
+test_files: []