fluent-plugin-concat 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 27d3958d3254219e7ca4fe4e063ba5d9fcccadd9
+  data.tar.gz: bf836cffc244561d46bd47837aa3ce0018c36c93
+SHA512:
+  metadata.gz: 44fa6336681c18243a889ac5e0124f91d41654c8df7f4c8af65d9796bc5498805431035b841e2013a430131f7e4e63188e5caa10ea71b7150ff4fbeb357fb77b
+  data.tar.gz: 298996442014f001cd197a051231278d59f5877402c1993e627e88840e25d0af158dda3da2294b3405d78ed9d3d7d36037cc74c61b6c8fa493c784c0f4e0f598

data/.gitignore

@@ -0,0 +1,6 @@
+/.bundle/
+/Gemfile.lock
+/coverage/
+/doc/
+/pkg/
+/tmp/

data/.rubocop.yml

@@ -0,0 +1,52 @@
+AllCops:
+  TargetRubyVersion: 2.3
+  DisplayCopNames: true
+  Exclude:
+    - "vendor/**/*"
+
+Lint/UnderscorePrefixedVariableName:
+  Enabled: false
+Lint/UnusedMethodArgument:
+  Enabled: false
+
+Metrics/LineLength:
+  Enabled: false
+Metrics/ClassLength:
+  Exclude:
+    - "test/**/*.rb"
+Metrics/ParameterLists:
+  Max: 7
+Metrics/MethodLength:
+  Max: 30
+
+Style/AsciiComments:
+  Enabled: false
+Style/HashSyntax:
+  Exclude:
+    - "**/*.rake"
+    - "Rakefile"
+Style/FrozenStringLiteralComment:
+  Enabled: false
+Style/StringLiterals:
+  EnforcedStyle: double_quotes
+Style/TrailingCommaInLiteral:
+  Enabled: false
+Style/TrailingCommaInArguments:
+  Enabled: false
+Style/FileName:
+  Enabled: false
+Style/Documentation:
+  Enabled: false
+Style/WordArray:
+  Enabled: false
+Style/BarePercentLiterals:
+  EnforcedStyle: percent_q
+Style/SpaceInsideBlockBraces:
+  EnforcedStyle: space
+  SpaceBeforeBlockParameters: false
+Style/BracesAroundHashParameters:
+  Enabled: false
+Style/MultilineMethodCallIndentation:
+  EnforcedStyle: indented
+Style/DoubleNegation:
+  Enabled: false

data/.travis.yml

@@ -0,0 +1,7 @@
+sudo: false
+language: ruby
+
+rvm:
+  - 2.1.8
+  - 2.2.4
+  - 2.3.0

data/Gemfile

@@ -0,0 +1,4 @@
+source "https://rubygems.org"
+
+# Specify your gem's dependencies in fluent-plugin-concat.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2016 okkez
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,77 @@
+# fluent-plugin-concat
+
+[![Build Status](https://travis-ci.org/okkez/fluent-plugin-concat.svg?branch=master)](https://travis-ci.org/okkez/fluent-plugin-concat)
+
+Fluentd Filter plugin to concatenate multiline log separated in multiple events.
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'fluent-plugin-concat'
+```
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install fluent-plugin-concat
+
+## Configuration
+
+**key** (required)
+
+The key for part of multiline log.
+
+**separator**
+
+The separator of lines.
+
+**n\_lines**
+
+The number of lines.
+This is exclusive with `multiline_start_regex`.
+
+**multiline\_start\_regexp**
+
+The regexp to match beginning of multiline.
+This is exclusive with `n_lines.`
+
+
+## Usage
+
+Every 10 events will be concatenated into one event.
+
+```aconf
+<filter docker.log>
+  @type concat
+  key message
+  n_lines 10
+</filter>
+```
+
+Specify first line of multiline by regular expression.
+
+```aconf
+<filter docker.log>
+  @type concat
+  key message
+  multiline_start_regexp /^Start/
+</filter>
+```
+
+## Contributing
+
+1. Fork it
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new Pull Request
+
+## License
+
+The gem is available as open source under the terms of the [MIT License](http://opensource.org/licenses/MIT).
+

data/Rakefile

@@ -0,0 +1,10 @@
+require "bundler/gem_tasks"
+require "rake/testtask"
+task :default => :test
+
+Rake::TestTask.new(:test) do |t|
+  t.libs << "test"
+  t.test_files = Dir["test/**/test_*.rb"].sort
+  t.verbose = false
+  t.warning = false
+end

data/fluent-plugin-concat.gemspec

@@ -0,0 +1,23 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+
+Gem::Specification.new do |spec|
+  spec.name          = "fluent-plugin-concat"
+  spec.version       = "0.1.0"
+  spec.authors       = ["Kenji Okimoto"]
+  spec.email         = ["okimoto@clear-code.com"]
+
+  spec.summary       = "Fluentd Filter plugin to concat multiple event messages"
+  spec.description   = "Fluentd Filter plugin to concat multiple event messages"
+  spec.homepage      = "https://github.com/okkez/fluent-plugin-concat"
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files -z`.split("\x0").reject {|f| f.match(%r{^(test|spec|features)/}) }
+  spec.require_paths = ["lib"]
+
+  spec.add_runtime_dependency "fluentd", "~> 0.12.21"
+  spec.add_development_dependency "bundler", "~> 1.11"
+  spec.add_development_dependency "rake", "~> 10.0"
+  spec.add_development_dependency "test-unit", ">= 3.1.0"
+end

data/lib/fluent/plugin/filter_concat.rb

@@ -0,0 +1,88 @@
+module Fluent
+  class ConcatFilter < Filter
+    Plugin.register_filter("concat", self)
+
+    desc "The key for part of multiline log"
+    config_param :key, :string, required: true
+    desc "The separator of lines"
+    config_param :separator, :string, default: "\n"
+    desc "The number of lines"
+    config_param :n_lines, :integer, default: nil
+    desc "The regexp to match beginning of multiline"
+    config_param :multiline_start_regexp, :string, default: nil
+
+    def initialize
+      super
+
+      @buffer = []
+    end
+
+    def configure(conf)
+      super
+
+      if @n_lines && @multiline_start_regexp
+        raise ConfigError, "n_lines and multiline_start_regexp are exclusive"
+      end
+      if @n_lines.nil? && @multiline_start_regexp.nil?
+        raise ConfigError, "Either n_lines or multiline_start_regexp is required"
+      end
+
+      @mode = nil
+      case
+      when @n_lines
+        @mode = :line
+      when @multiline_start_regexp
+        @multiline_start_regexp = Regexp.compile(@multiline_start_regexp[1..-2])
+        @mode = :regexp
+      end
+    end
+
+    def filter_stream(tag, es)
+      new_es = MultiEventStream.new
+      es.each do |time, record|
+        begin
+          new_record = process(record)
+          new_es.add(time, record.merge(new_record)) if new_record
+        rescue => e
+          router.emit_error_event(tag, time, record, e)
+        end
+      end
+      new_es
+    end
+
+    private
+
+    def process(record)
+      case @mode
+      when :line
+        @buffer << record[@key]
+        if @n_lines && @buffer.size >= @n_lines
+          return flush_buffer
+        end
+      when :regexp
+        if firstline?(record[@key])
+          if @buffer.empty?
+            @buffer << record[@key]
+          else
+            return flush_buffer(record[@key])
+          end
+        else
+          @buffer << record[@key]
+        end
+      end
+      nil
+    end
+
+    def firstline?(text)
+      !!@multiline_start_regexp.match(text)
+    end
+
+    def flush_buffer(new_message = nil)
+      new_record = {}
+      new_record[@key] = @buffer.join(@separator)
+      @buffer = []
+      @buffer << new_message if new_message
+      new_record
+    end
+  end
+end

metadata

@@ -0,0 +1,109 @@
+--- !ruby/object:Gem::Specification
+name: fluent-plugin-concat
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Kenji Okimoto
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2016-04-07 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: fluentd
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.12.21
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.12.21
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.11'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.11'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+- !ruby/object:Gem::Dependency
+  name: test-unit
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 3.1.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 3.1.0
+description: Fluentd Filter plugin to concat multiple event messages
+email:
+- okimoto@clear-code.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".rubocop.yml"
+- ".travis.yml"
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- fluent-plugin-concat.gemspec
+- lib/fluent/plugin/filter_concat.rb
+homepage: https://github.com/okkez/fluent-plugin-concat
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.5.1
+signing_key: 
+specification_version: 4
+summary: Fluentd Filter plugin to concat multiple event messages
+test_files: []