fluent-plugin-concat 2.3.0 → 2.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: ee5e1936b898d90227550ba4baadba4f3d64d31577a1b0923aa634a0b3a49e6c
-  data.tar.gz: 59be67a9067f3589e9728702a4b4a8bb9f25e8915eedd4aa308dc297e710ba46
+  metadata.gz: d906b3c9372b21adec5b1515e4820b037d90a0fa83a32f01dad1a8d6e5e0f79c
+  data.tar.gz: 9685f06e791e53ae0fcb3bb3e71991f812405298084c31cb4cc61a4181e17d6c
 SHA512:
-  metadata.gz: 2827392627d1d6d1eba547bb1d167fd86d9ebb88cb0c5aa98db6cb124a254da0af841de55a9ef4f437548a6c4175202f8bd751647ee711d40b2fd3607400f829
-  data.tar.gz: c827c2d9b14b509abd801428ac9aae29f34d99b12e6359e2152eb4b4716a7204dddf3638a9e9124da0eaee0d17fb8edd5e2ff18e6c3ff7fba5be4b000fad097f
+  metadata.gz: 1683fcfe93f86fedbc89334dde6565b48fad41800631a5624ebfd26f89f8c725c0cff1aed9ef3733d863bf1ec3fb2e0e62676ce78a68dc40d332b08f80209748
+  data.tar.gz: 6bec85d758dfce0dbced83a03413d2e1c2cef6d73a9c30ccb07f03c62d5d211d13559fdd7703f373f72af6d3c631ecf753451b93737777e82b2fc053ae5cb0ef

data/.rubocop.yml

@@ -4,6 +4,11 @@ AllCops:
   Exclude:
     - "vendor/**/*"
 
+Layout/MultilineMethodCallIndentation:
+  EnforcedStyle: indented
+Layout/SpaceInsideBlockBraces:
+  EnforcedStyle: space
+  SpaceBeforeBlockParameters: false
 Lint/UnderscorePrefixedVariableName:
   Enabled: false
 Lint/UnusedMethodArgument:
@@ -19,6 +24,9 @@ Metrics/ParameterLists:
 Metrics/MethodLength:
   Max: 30
 
+Naming/FileName:
+  Enabled: false
+
 Style/AsciiComments:
   Enabled: false
 Style/HashSyntax:
@@ -29,11 +37,11 @@ Style/FrozenStringLiteralComment:
   Enabled: false
 Style/StringLiterals:
   EnforcedStyle: double_quotes
-Style/TrailingCommaInLiteral:
+Style/TrailingCommaInArrayLiteral:
   Enabled: false
-Style/TrailingCommaInArguments:
+Style/TrailingCommaInHashLiteral:
   Enabled: false
-Style/FileName:
+Style/TrailingCommaInArguments:
   Enabled: false
 Style/Documentation:
   Enabled: false
@@ -41,13 +49,8 @@ Style/WordArray:
   Enabled: false
 Style/BarePercentLiterals:
   EnforcedStyle: percent_q
-Style/SpaceInsideBlockBraces:
-  EnforcedStyle: space
-  SpaceBeforeBlockParameters: false
 Style/BracesAroundHashParameters:
   Enabled: false
-Style/MultilineMethodCallIndentation:
-  EnforcedStyle: indented
 Style/DoubleNegation:
   Enabled: false
 Style/EmptyCaseCondition:

data/.travis.yml

@@ -9,13 +9,11 @@ notifications:
     on_failure: always
     on_start: never
 
-rvm:
-  - 2.1
-  - 2.2
-  - 2.3
-  - 2.4.3
-  - 2.5.0
-
-gemfile:
-  - Gemfile
-  - gemfiles/fluentd_v0.14.gemfile
+matrix:
+  include:
+    - rvm: 2.4.5
+      gemfile: Gemfile
+    - rvm: 2.5.5
+      gemfile: Gemfile
+    - rvm: 2.6.2
+      gemfile: Gemfile

data/Appraisals

@@ -1,3 +0,0 @@
-appraise "fluentd v0.14" do
-  gem "fluentd", "~> 0.14.0"
-end

data/README.md

@@ -33,58 +33,69 @@ Or install it yourself as:
 
 The key for part of multiline log.
 
-**separator** (string)
+**separator** (string) (optional)
 
 The separator of lines.
 Default value is `"\n"`.
 
-**n\_lines** (integer)
+**n\_lines** (integer) (optional)
 
 The number of lines.
 This is exclusive with `multiline_start_regex`.
 
-**multiline\_start\_regexp**
+**multiline\_start\_regexp** (string) (optional)
 
 The regexp to match beginning of multiline.
 This is exclusive with `n_lines.`
 
-**multiline\_end\_regexp**
+**multiline\_end\_regexp** (string) (optional)
 
 The regexp to match ending of multiline.
 This is exclusive with `n_lines.`
 
-**continuous\_line\_regexp**
+**continuous\_line\_regexp** (string) (optional)
 
 The regexp to match continuous lines.
 This is exclusive with `n_lines.`
 
-**stream\_identity\_key** (string)
+**stream\_identity\_key** (string) (optional)
 
 The key to determine which stream an event belongs to.
 
-**flush\_interval** (integer)
+**flush\_interval** (integer) (optional)
 
 The number of seconds after which the last received event log will be flushed.
 If specified 0, wait for next line forever.
 
-**use\_first\_timestamp** (bool)
+**timeout\_label** (string) (optional)
+
+The label name to handle events caused by timeout.
+
+**use\_first\_timestamp** (bool) (optional)
 
 Use timestamp of first record when buffer is flushed.
 Default value is `false`.
 
-**partial\_key** (string)
+**partial\_key** (string) (optional)
 
 The field name that is the reference to concatenate records
 
-**partial\_value** (string)
+**partial\_value** (string) (optional)
 
 The value stored in the field specified by partial_key that represent partial log
 
-**keep\_partial\_key** (bool)
+**keep\_partial\_key** (bool) (optional)
 
 If true, keep partial_key in concatenated records
 Default value is `false`.
 
+**use\_partial\_metadata** (bool) (optional)
+
+Use partial metadata to concatenate multiple records
+
+**keep\_partial\_metadata** (bool) (optional)
+
+If true, keep partial metadata
 
 ## Usage
 
@@ -152,7 +163,7 @@ Handle single line JSON from Docker containers.
 </filter>
 ```
 
-Handle Docker's `partial_message`.
+Handle Docker logs splitted in several parts (using `partial_message`), and do not add new line between parts.
 
 ```aconf
 <filter>
@@ -160,6 +171,18 @@ Handle Docker's `partial_message`.
   key message
   partial_key partial_message
   partial_value true
+  separator ""
+</filter>
+```
+
+Handle Docker logs splitted in several parts (using newline detection), and do not add new line between parts (prior to Docker 18.06).
+
+```aconf
+<filter **>
+  @type concat
+  key log
+  multiline_end_regexp /\\n$/
+  separator ""
 </filter>
 ```
 

data/fluent-plugin-concat.gemspec

@@ -4,7 +4,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |spec|
   spec.name          = "fluent-plugin-concat"
-  spec.version       = "2.3.0"
+  spec.version       = "2.4.0"
   spec.authors       = ["Kenji Okimoto"]
   spec.email         = ["okimoto@clear-code.com"]
 
@@ -17,7 +17,7 @@ Gem::Specification.new do |spec|
   spec.require_paths = ["lib"]
 
   spec.add_runtime_dependency "fluentd", ">= 0.14.0", "< 2"
-  spec.add_development_dependency "bundler", "~> 1.11"
+  spec.add_development_dependency "bundler"
   spec.add_development_dependency "rake", "~> 12.0"
   spec.add_development_dependency "test-unit", ">= 3.1.0"
   spec.add_development_dependency "test-unit-rr"

data/lib/fluent/plugin/filter_concat.rb

@@ -32,6 +32,10 @@ module Fluent::Plugin
     config_param :partial_value, :string, default: nil
     desc "If true, keep partial_key in concatenated records"
     config_param :keep_partial_key, :bool, default: false
+    desc "Use partial metadata to concatenate multiple records"
+    config_param :use_partial_metadata, :bool, default: false
+    desc "If true, keep partial metadata"
+    config_param :keep_partial_metadata, :bool, default: false
 
     class TimeoutError < StandardError
     end
@@ -49,12 +53,12 @@ module Fluent::Plugin
     def configure(conf)
       super
 
+      if @n_lines.nil? && @multiline_start_regexp.nil? && @multiline_end_regexp.nil? && @partial_key.nil? && !@use_partial_metadata
+        raise Fluent::ConfigError, "Either n_lines, multiline_start_regexp, multiline_end_regexp, partial_key or use_partial_metadata is required"
+      end
       if @n_lines && (@multiline_start_regexp || @multiline_end_regexp)
         raise Fluent::ConfigError, "n_lines and multiline_start_regexp/multiline_end_regexp are exclusive"
       end
-      if @n_lines.nil? && @multiline_start_regexp.nil? && @multiline_end_regexp.nil? && @partial_key.nil?
-        raise Fluent::ConfigError, "Either n_lines or multiline_start_regexp or multiline_end_regexp is required"
-      end
       if @partial_key && @n_lines
         raise Fluent::ConfigError, "partial_key and n_lines are exclusive"
       end
@@ -64,6 +68,15 @@ module Fluent::Plugin
       if @partial_key && @partial_value.nil?
         raise Fluent::ConfigError, "partial_value is required when partial_key is specified"
       end
+      if @use_partial_metadata && @n_lines
+        raise Fluent::ConfigError, "user_partial_metadata and n_lines are exclusive"
+      end
+      if @use_partial_metadata && (@multiline_start_regexp || @multiline_end_regexp)
+        raise Fluent::ConfigError, "user_partial_metadata and multiline_start_regexp/multiline_end_regexp are exclusive"
+      end
+      if @use_partial_metadata && @partial_key
+        raise Fluent::ConfigError, "user_partial_metadata and partial_key are exclusive"
+      end
 
       @mode = nil
       case
@@ -71,6 +84,8 @@ module Fluent::Plugin
         @mode = :line
       when @partial_key
         @mode = :partial
+      when @use_partial_metadata
+        @mode = :partial_metadata
       when @multiline_start_regexp || @multiline_end_regexp
         @mode = :regexp
         if @multiline_start_regexp
@@ -108,13 +123,35 @@ module Fluent::Plugin
           new_es.add(time, record)
           next
         end
+        if @mode == :partial
+          unless record.key?(@partial_key)
+            new_es.add(time, record)
+            next
+          end
+        end
+        if @mode == :partial_metadata
+          unless record.key?("partial_message")
+            new_es.add(time, record)
+            next
+          end
+        end
         begin
           flushed_es = process(tag, time, record)
           unless flushed_es.empty?
             flushed_es.each do |_time, new_record|
               time = _time if @use_first_timestamp
               merged_record = record.merge(new_record)
-              merged_record.delete(@partial_key) unless @keep_partial_key
+              case @mode
+              when :partial
+                merged_record.delete(@partial_key) unless @keep_partial_key
+              when :partial_metadata
+                unless @keep_partial_metadata
+                  merged_record.delete("partial_message")
+                  merged_record.delete("partial_id")
+                  merged_record.delete("partial_ordinal")
+                  merged_record.delete("partial_last")
+                end
+              end
               new_es.add(time, merged_record)
             end
           end
@@ -136,10 +173,18 @@ module Fluent::Plugin
     end
 
     def process(tag, time, record)
-      if @stream_identity_key
-        stream_identity = "#{tag}:#{record[@stream_identity_key]}"
+      if @mode == :partial_metadata
+        if @stream_identity_key
+          stream_identity = %Q(#{tag}:#{record[@stream_identity_key]}#{record["partial_id"]})
+        else
+          stream_identity = %Q(#{tag}:#{record["partial_id"]})
+        end
       else
-        stream_identity = "#{tag}:default"
+        if @stream_identity_key
+          stream_identity = "#{tag}:#{record[@stream_identity_key]}"
+        else
+          stream_identity = "#{tag}:default"
+        end
       end
       @timeout_map_mutex.synchronize do
         @timeout_map[stream_identity] = Fluent::Engine.now
@@ -149,6 +194,8 @@ module Fluent::Plugin
         process_line(stream_identity, tag, time, record)
       when :partial
         process_partial(stream_identity, tag, time, record)
+      when :partial_metadata
+        process_partial_metadata(stream_identity, tag, time, record)
       when :regexp
         process_regexp(stream_identity, tag, time, record)
       end
@@ -177,6 +224,18 @@ module Fluent::Plugin
       new_es
     end
 
+    def process_partial_metadata(stream_identity, tag, time, record)
+      new_es = Fluent::MultiEventStream.new
+      @buffer[stream_identity] << [tag, time, record]
+      if record["partial_last"] == "true"
+        new_time, new_record = flush_buffer(stream_identity)
+        time = new_time if @use_first_timestamp
+        new_record.delete(@partial_key)
+        new_es.add(time, new_record)
+      end
+      new_es
+    end
+
     def process_regexp(stream_identity, tag, time, record)
       new_es = Fluent::MultiEventStream.new
       case
@@ -245,7 +304,13 @@ module Fluent::Plugin
     end
 
     def flush_buffer(stream_identity, new_element = nil)
-      lines = @buffer[stream_identity].map {|_tag, _time, record| record[@key] }
+      lines = if @mode == :partial_metadata
+                @buffer[stream_identity]
+                  .sort_by {|_tag, _time, record| record["partial_ordinal"].to_i }
+                  .map {|_tag, _time, record| record[@key] }
+              else
+                @buffer[stream_identity].map {|_tag, _time, record| record[@key] }
+              end
       _tag, time, first_record = @buffer[stream_identity].first
       new_record = {
         @key => lines.join(@separator)

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-concat
 version: !ruby/object:Gem::Version
-  version: 2.3.0
+  version: 2.4.0
 platform: ruby
 authors:
 - Kenji Okimoto
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-07-23 00:00:00.000000000 Z
+date: 2019-07-26 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -34,16 +34,16 @@ dependencies:
   name: bundler
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '1.11'
+        version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '1.11'
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
@@ -118,7 +118,6 @@ files:
 - README.md
 - Rakefile
 - fluent-plugin-concat.gemspec
-- gemfiles/fluentd_v0.14.gemfile
 - lib/fluent/plugin/filter_concat.rb
 homepage: https://github.com/fluent-plugins-nursery/fluent-plugin-concat
 licenses:
@@ -139,8 +138,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.7.6
+rubygems_version: 3.0.3
 signing_key: 
 specification_version: 4
 summary: Fluentd Filter plugin to concat multiple event messages

data/gemfiles/fluentd_v0.14.gemfile

@@ -1,7 +0,0 @@
-# This file was generated by Appraisal
-
-source "https://rubygems.org"
-
-gem "fluentd", "~> 0.14.0"
-
-gemspec :path => "../"