fluent-plugin-concat 2.2.3 → 2.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 0a119cbc6b5774f81bde44b530ece7292c1f83635d6577155baad7d3c6d9544c
-  data.tar.gz: 92c703271e63894f9b56c8a4d3d22c68c79c2ee6e219adb9b934fbb05a7464cf
+  metadata.gz: ee5e1936b898d90227550ba4baadba4f3d64d31577a1b0923aa634a0b3a49e6c
+  data.tar.gz: 59be67a9067f3589e9728702a4b4a8bb9f25e8915eedd4aa308dc297e710ba46
 SHA512:
-  metadata.gz: cc59b890e0a82eca4a27663f2705f1ae4bf98e2d89c626696f90f523b73310df779cb26b92fdf544aa0e149f3c3e4b97569ea7291475b828ed2378082ec3a0bf
-  data.tar.gz: 6b7eca55c66e0a038faf738ff718952a57bbf382881ffc7c03663984ede6f8e34d3219b737f4922ec83e1e82d82d823d64ad4a943e96dc9d8581fac1c78a81ad
+  metadata.gz: 2827392627d1d6d1eba547bb1d167fd86d9ebb88cb0c5aa98db6cb124a254da0af841de55a9ef4f437548a6c4175202f8bd751647ee711d40b2fd3607400f829
+  data.tar.gz: c827c2d9b14b509abd801428ac9aae29f34d99b12e6359e2152eb4b4716a7204dddf3638a9e9124da0eaee0d17fb8edd5e2ff18e6c3ff7fba5be4b000fad097f

data/NEWS.md

@@ -1,5 +1,11 @@
 # ChangeLog
 
+## v2.3.0
+
+### Improvements
+
+* Support `partial_key`, `partial_value` to concatenate split log lines. See #46, #52
+
 ## v2.2.3
 
 ### Fixes

data/README.md

@@ -29,16 +29,16 @@ Or install it yourself as:
 
 ## Configuration
 
-**key** (required)
+**key** (string) (required)
 
 The key for part of multiline log.
 
-**separator**
+**separator** (string)
 
 The separator of lines.
 Default value is `"\n"`.
 
-**n\_lines**
+**n\_lines** (integer)
 
 The number of lines.
 This is exclusive with `multiline_start_regex`.
@@ -58,18 +58,33 @@ This is exclusive with `n_lines.`
 The regexp to match continuous lines.
 This is exclusive with `n_lines.`
 
-**stream\_identity\_key**
+**stream\_identity\_key** (string)
 
 The key to determine which stream an event belongs to.
 
-**flush\_interval**
+**flush\_interval** (integer)
 
 The number of seconds after which the last received event log will be flushed.
 If specified 0, wait for next line forever.
 
-**use\_first\_timestamp**
+**use\_first\_timestamp** (bool)
 
 Use timestamp of first record when buffer is flushed.
+Default value is `false`.
+
+**partial\_key** (string)
+
+The field name that is the reference to concatenate records
+
+**partial\_value** (string)
+
+The value stored in the field specified by partial_key that represent partial log
+
+**keep\_partial\_key** (bool)
+
+If true, keep partial_key in concatenated records
+Default value is `false`.
+
 
 ## Usage
 
@@ -137,6 +152,51 @@ Handle single line JSON from Docker containers.
 </filter>
 ```
 
+Handle Docker's `partial_message`.
+
+```aconf
+<filter>
+  @type concat
+  key message
+  partial_key partial_message
+  partial_value true
+</filter>
+```
+
+Handle containerd/cri in Kubernetes.
+
+```aconf
+<source>
+  @type tail
+  path /var/log/containers/*.log
+  <parse>
+    @type regexp
+    expression /^(?<timestamp>\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d+Z) (?<output>\w+) (?<partial_flag>[FP]) (?<message>.+)$/
+  </parse>
+  tag k8s
+  @label @CONCAT
+</source>
+
+<label @CONCAT>
+  <filter k8s>
+    @type concat
+    key message
+    partial_key partial_flag
+    partial_value P
+  </filter>
+  <match k8s>
+    @type relabel
+    @label @OUTPUT
+  </match>
+</label>
+
+<label @OUTPUT>
+  <match>
+    @type stdout
+  </match>
+</label>
+```
+
 ## Contributing
 
 1. Fork it

data/fluent-plugin-concat.gemspec

@@ -4,7 +4,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |spec|
   spec.name          = "fluent-plugin-concat"
-  spec.version       = "2.2.3"
+  spec.version       = "2.3.0"
   spec.authors       = ["Kenji Okimoto"]
   spec.email         = ["okimoto@clear-code.com"]
 

data/lib/fluent/plugin/filter_concat.rb

@@ -26,6 +26,12 @@ module Fluent::Plugin
     config_param :timeout_label, :string, default: nil
     desc "Use timestamp of first record when buffer is flushed"
     config_param :use_first_timestamp, :bool, default: false
+    desc "The field name that is the reference to concatenate records"
+    config_param :partial_key, :string, default: nil
+    desc "The value stored in the field specified by partial_key that represent partial log"
+    config_param :partial_value, :string, default: nil
+    desc "If true, keep partial_key in concatenated records"
+    config_param :keep_partial_key, :bool, default: false
 
     class TimeoutError < StandardError
     end
@@ -43,17 +49,28 @@ module Fluent::Plugin
     def configure(conf)
       super
 
-      if @n_lines && (@multiline_start_regexp || @multiline_end_regexp || @continuous_line_regexp)
-        raise Fluent::ConfigError, "n_lines and multiline_start_regexp/multiline_end_regexp/continuous_line_regexp are exclusive"
+      if @n_lines && (@multiline_start_regexp || @multiline_end_regexp)
+        raise Fluent::ConfigError, "n_lines and multiline_start_regexp/multiline_end_regexp are exclusive"
       end
-      if @n_lines.nil? && @multiline_start_regexp.nil? && @multiline_end_regexp.nil?
+      if @n_lines.nil? && @multiline_start_regexp.nil? && @multiline_end_regexp.nil? && @partial_key.nil?
         raise Fluent::ConfigError, "Either n_lines or multiline_start_regexp or multiline_end_regexp is required"
       end
+      if @partial_key && @n_lines
+        raise Fluent::ConfigError, "partial_key and n_lines are exclusive"
+      end
+      if @partial_key && (@multiline_start_regexp || @multiline_end_regexp)
+        raise Fluent::ConfigError, "partial_key and multiline_start_regexp/multiline_end_regexp are exclusive"
+      end
+      if @partial_key && @partial_value.nil?
+        raise Fluent::ConfigError, "partial_value is required when partial_key is specified"
+      end
 
       @mode = nil
       case
       when @n_lines
         @mode = :line
+      when @partial_key
+        @mode = :partial
       when @multiline_start_regexp || @multiline_end_regexp
         @mode = :regexp
         if @multiline_start_regexp
@@ -96,7 +113,9 @@ module Fluent::Plugin
           unless flushed_es.empty?
             flushed_es.each do |_time, new_record|
               time = _time if @use_first_timestamp
-              new_es.add(time, record.merge(new_record))
+              merged_record = record.merge(new_record)
+              merged_record.delete(@partial_key) unless @keep_partial_key
+              new_es.add(time, merged_record)
             end
           end
         rescue => e
@@ -128,6 +147,8 @@ module Fluent::Plugin
       case @mode
       when :line
         process_line(stream_identity, tag, time, record)
+      when :partial
+        process_partial(stream_identity, tag, time, record)
       when :regexp
         process_regexp(stream_identity, tag, time, record)
       end
@@ -144,6 +165,18 @@ module Fluent::Plugin
       new_es
     end
 
+    def process_partial(stream_identity, tag, time, record)
+      new_es = Fluent::MultiEventStream.new
+      @buffer[stream_identity] << [tag, time, record]
+      unless @partial_value == record[@partial_key]
+        new_time, new_record = flush_buffer(stream_identity)
+        time = new_time if @use_first_timestamp
+        new_record.delete(@partial_key)
+        new_es.add(time, new_record)
+      end
+      new_es
+    end
+
     def process_regexp(stream_identity, tag, time, record)
       new_es = Fluent::MultiEventStream.new
       case

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-concat
 version: !ruby/object:Gem::Version
-  version: 2.2.3
+  version: 2.3.0
 platform: ruby
 authors:
 - Kenji Okimoto
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-06-15 00:00:00.000000000 Z
+date: 2018-07-23 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd