fluent-plugin-detect-exceptions 0.0.13 → 0.0.14

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 3fa3afee1139adec8fe98c8406bd72035d844aadf9f1715c01c5db3e45bd7e1c
-  data.tar.gz: 63b1bba51e317c9931d4904551b64ef94c56b0946bccec6f67edc2311c0bbba2
+  metadata.gz: f846941463ccc37acd71280e182d5ae7b0ffd0c95c609bfbfe67de64f86ff478
+  data.tar.gz: 1454c4ddae5ba4efa214972f2109f0a71d4984e87b49cefa3541cb87d92d6cc7
 SHA512:
-  metadata.gz: 60bbd111f5818ebfa4eda68c87236c3d985df25e66acf12e29f2851f52c4f9e0c74a0d8bbeeb5f077f1b154c4f23fa2bb223ac26f230aff29fdc5d0ddfa69272
-  data.tar.gz: 85e20f53eb5e72b7062b6d60d724f55edc53997530dce6ca6adc252bb422a7c5466c73dc5370909ec944a52f7097a8d632649b2ed32fb16962689073dfe3415b
+  metadata.gz: 585f0de72a4f845f39c0f7e753ca3661ad335b10c18d26930afc19bab246b205cb9d68d1fa506c87e5ea674fc9b1ce51b04c278aed5590d997875ea9b50b21d8
+  data.tar.gz: 987c453e5667c1736507500859a1f46612a56451f791afaf194574380afd31527da73f9ebd87de53faf2c40e5929ce0cdd0f2c3fbd0c9d4c89a2137bad40a9a1

data/README.rdoc

@@ -42,6 +42,16 @@ will also install and configure the gem.
 
 The plugin supports the following parameters:
 
+=== Required
+
+[remove_tag_prefix]  The prefix to remove from the input tag when outputting
+                     a record. A prefix has to be a complete tag part.
+                     Example: If remove_tag_prefix is set to 'foo', the input
+                     tag foo.bar.baz is transformed to bar.baz and the input tag
+                     'foofoo.bar' is not modified.
+
+=== Optional
+
 [message]  Name of the field in the JSON record that contains the
            single-line log messages that shall be scanned for exceptions.
            If this is set to '', the plugin will try 'message' and 'log',
@@ -49,12 +59,6 @@ The plugin supports the following parameters:
            This parameter is only applicable to structured (JSON) log streams.
            Default: ''.
 
-[remove_tag_prefix]  The prefix to remove from the input tag when outputting
-                     a record. A prefix has to be a complete tag part.
-                     Example: If remove_tag_prefix is set to 'foo', the input
-                     tag foo.bar.baz is transformed to bar.baz and the input tag
-                     'foofoo.bar' is not modified. Default: empty string.
-
 [languages]  A list of language for which exception stack traces shall be
              detected. The values in the list can be separated by commas or
              written as JSON list.
@@ -66,6 +70,12 @@ The plugin supports the following parameters:
                             forwarded. If not set, incomplete exceptions stacks
                             are not flushed.
 
+[force_line_breaks]  Force line breaks between each lines when comibining exception stacks.
+                     This is useful if your exception is formatted
+                     as a single line. i.e., logs retrieved from the docker's
+                     logging driver don't have any line break.
+                     Default: false.
+
 [max_lines]  Maximum number of lines in a detected exception stack trace.
              If this maximum number is exceeded, the exception stack trace
              that has been detected so far will be output as a single

data/fluent-plugin-detect-exceptions.gemspec

@@ -11,7 +11,7 @@ eos
   gem.homepage      = \
     'https://github.com/GoogleCloudPlatform/fluent-plugin-detect-exceptions'
   gem.license       = 'Apache-2.0'
-  gem.version       = '0.0.13'
+  gem.version       = '0.0.14'
   gem.authors       = ['Stackdriver Agents']
   gem.email         = ['stackdriver-agents@google.com']
   gem.required_ruby_version = Gem::Requirement.new('>= 2.0')

data/lib/fluent/plugin/exception_detector.rb

@@ -256,20 +256,22 @@ module Fluent
     # message_field may contain the empty string. In this case, the
     # TraceAccumulator 'learns' the field name from the first record by checking
     # for some pre-defined common field names of text logs.
-    # The named parameters max_lines and max_bytes limit the maximum amount
+    # The option parameter can be used to pass the following parameters:
+    # force_line_breaks adds line breaks when combining exception stacks
+    # max_lines and max_bytes limit the maximum amount
     # of data to be buffered. The default value 0 indicates 'no limit'.
-    def initialize(message_field, languages, max_lines: 0, max_bytes: 0,
-                   &emit_callback)
+    def initialize(message_field, languages, **options, &emit_callback)
       @exception_detector = Fluent::ExceptionDetector.new(*languages)
-      @max_lines = max_lines
-      @max_bytes = max_bytes
       @message_field = message_field
+      @force_line_breaks = options[:force_line_breaks] || false
+      @max_lines = options[:max_lines]           || 0
+      @max_bytes = options[:max_bytes]           || 0
+      @emit = emit_callback
       @messages = []
       @buffer_start_time = Time.now
       @buffer_size = 0
       @first_record = nil
       @first_timestamp = nil
-      @emit = emit_callback
     end
 
     def push(time_sec, record)
@@ -360,8 +362,14 @@ module Fluent
         @buffer_start_time = Time.now
       end
       unless message.nil?
-        @messages << message
-        @buffer_size += message.length
+        message_with_line_break =
+          if @force_line_breaks && !@messages.empty? && !message.include?("\n")
+            "\n" + message
+          else
+            message
+          end
+        @messages << message_with_line_break
+        @buffer_size += message_with_line_break.length
       end
     end
   end

data/lib/fluent/plugin/out_detect_exceptions.rb

@@ -22,14 +22,16 @@ module Fluent
   # an exception stack trace, they forwarded as a single, combined JSON
   # object. Otherwise, the input log data is forwarded as is.
   class DetectExceptionsOutput < Output
+    desc 'The prefix to be removed from the input tag when outputting a record.'
+    config_param :remove_tag_prefix, :string
     desc 'The field which contains the raw message text in the input JSON data.'
     config_param :message, :string, default: ''
-    desc 'The prefix to be removed from the input tag when outputting a record.'
-    config_param :remove_tag_prefix, :string, default: ''
     desc 'The interval of flushing the buffer for multiline format.'
     config_param :multiline_flush_interval, :time, default: nil
     desc 'Programming languages for which to detect exceptions. Default: all.'
     config_param :languages, :array, value_type: :string, default: []
+    desc 'Force live breaks when combining exception stacks. Default: false.'
+    config_param :force_line_breaks, :bool, default: false
     desc 'Maximum number of lines to flush (0 means no limit). Default: 1000.'
     config_param :max_lines, :integer, default: 1000
     desc 'Maximum number of bytes to flush (0 means no limit). Default: 0.'
@@ -91,9 +93,13 @@ module Fluent
         unless @accumulators.key?(log_id)
           out_tag = tag.sub(/^#{Regexp.escape(@remove_tag_prefix)}\./, '')
           @accumulators[log_id] =
-            Fluent::TraceAccumulator.new(@message, @languages,
-                                         max_lines: @max_lines,
-                                         max_bytes: @max_bytes) do |t, r|
+            Fluent::TraceAccumulator.new(
+              @message,
+              @languages,
+              force_line_breaks: @force_line_breaks,
+              max_lines: @max_lines,
+              max_bytes: @max_bytes
+            ) do |t, r|
               router.emit(out_tag, t, r)
             end
         end

data/test/plugin/test_out_detect_exceptions.rb

@@ -28,6 +28,8 @@ END
 
   DEFAULT_TAG = 'prefix.test.tag'.freeze
 
+  DEFAULT_TAG_STRIPPED = 'test.tag'.freeze
+
   ARBITRARY_TEXT = 'This line is not an exception.'.freeze
 
   JAVA_EXC = <<END.freeze
@@ -74,6 +76,17 @@ END
     log_entry
   end
 
+  def feed_lines_without_line_breaks(driver, t, *messages, stream: nil)
+    count = 0
+    messages.each do |m|
+      m.each_line do |line|
+        line.delete!("\n")
+        driver.emit(log_entry(line, count, stream), t + count)
+        count += 1
+      end
+    end
+  end
+
   def feed_lines(driver, t, *messages, stream: nil)
     count = 0
     messages.each do |m|
@@ -125,7 +138,9 @@ END
     }
 
     test_cases.each do |language, exception|
-      cfg = "languages #{language}"
+      cfg = %(
+#{CONFIG}
+languages #{language})
       d = create_driver(cfg)
       t = Time.now.to_i
 
@@ -156,12 +171,12 @@ END
 
       # Validate that each line received is emitted separately as expected.
       router_mock.should_receive(:emit)
-                 .once.with(DEFAULT_TAG, Integer,
+                 .once.with(DEFAULT_TAG_STRIPPED, Integer,
                             'message' => json_line_with_exception,
                             'count' => 0)
 
       router_mock.should_receive(:emit)
-                 .once.with(DEFAULT_TAG, Integer,
+                 .once.with(DEFAULT_TAG_STRIPPED, Integer,
                             'message' => json_line_without_exception,
                             'count' => 1)
 
@@ -174,7 +189,9 @@ END
   end
 
   def test_single_language_config
-    cfg = 'languages java'
+    cfg = %(
+#{CONFIG}
+languages java)
     d = create_driver(cfg)
     t = Time.now.to_i
     d.run do
@@ -185,7 +202,9 @@ END
   end
 
   def test_multi_language_config
-    cfg = 'languages python, java'
+    cfg = %(
+#{CONFIG}
+languages python, java)
     d = create_driver(cfg)
     t = Time.now.to_i
     d.run do
@@ -196,7 +215,9 @@ END
   end
 
   def test_split_exception_after_timeout
-    cfg = 'multiline_flush_interval 1'
+    cfg = %(
+#{CONFIG}
+multiline_flush_interval 1)
     d = create_driver(cfg)
     t1 = 0
     t2 = 0
@@ -227,6 +248,12 @@ END
     assert_equal(make_logs(t1, JAVA_EXC + "  at x\n  at y\n"), d.events)
   end
 
+  def test_remove_tag_prefix_is_required
+    cfg = ''
+    e = assert_raises(Fluent::ConfigError) { create_driver(cfg) }
+    assert_match(/remove_tag_prefix/, e.message)
+  end
+
   def get_out_tags(remove_tag_prefix, original_tag)
     cfg = "remove_tag_prefix #{remove_tag_prefix}"
     d = create_driver(cfg, original_tag)
@@ -243,8 +270,42 @@ END
     assert_equal(['prefix.plus.rest.of.the.tag'], tags)
   end
 
+  def test_force_line_breaks_false
+    cfg = %(
+#{CONFIG}
+force_line_breaks true)
+    d = create_driver(cfg)
+    t = Time.now.to_i
+    d.run do
+      feed_lines(d, t, JAVA_EXC)
+    end
+    expected = JAVA_EXC
+    assert_equal(make_logs(t, *expected), d.events)
+  end
+
+  def test_force_line_breaks_true
+    cfg = %(
+#{CONFIG}
+force_line_breaks true)
+    d = create_driver(cfg)
+    t = Time.now.to_i
+    d.run do
+      feed_lines_without_line_breaks(d, t, JAVA_EXC)
+    end
+    # Expected: the first two lines of the exception are buffered and combined.
+    # Then the max_lines setting kicks in and the rest of the Python exception
+    # is logged line-by-line (since it's not an exception stack in itself).
+    # For the following Java stack trace, the two lines of the first exception
+    # are buffered and combined. So are the first two lines of the second
+    # exception. Then the rest is logged line-by-line.
+    expected = JAVA_EXC.chomp
+    assert_equal(make_logs(t, *expected), d.events)
+  end
+
   def test_flush_after_max_lines
-    cfg = 'max_lines 2'
+    cfg = %(
+#{CONFIG}
+max_lines 2)
     d = create_driver(cfg)
     t = Time.now.to_i
     d.run do
@@ -263,7 +324,9 @@ END
   end
 
   def test_separate_streams
-    cfg = 'stream stream'
+    cfg = %(
+#{CONFIG}
+stream stream)
     d = create_driver(cfg)
     t = Time.now.to_i
     d.run do

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-detect-exceptions
 version: !ruby/object:Gem::Version
-  version: 0.0.13
+  version: 0.0.14
 platform: ruby
 authors:
 - Stackdriver Agents
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-11-04 00:00:00.000000000 Z
+date: 2021-09-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -93,7 +93,6 @@ extra_rdoc_files: []
 files:
 - CONTRIBUTING
 - Gemfile
-- Gemfile.lock
 - LICENSE
 - README.rdoc
 - Rakefile
@@ -123,7 +122,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.4
+rubygems_version: 3.0.8
 signing_key: 
 specification_version: 4
 summary: fluentd output plugin for combining stack traces as multi-line JSON logs

data/Gemfile.lock

@@ -1,66 +0,0 @@
-PATH
-  remote: .
-  specs:
-    fluent-plugin-detect-exceptions (0.0.13)
-      fluentd (>= 0.10)
-
-GEM
-  remote: https://rubygems.org/
-  specs:
-    ast (2.4.0)
-    concurrent-ruby (1.1.5)
-    cool.io (1.5.4)
-    dig_rb (1.0.1)
-    flexmock (2.3.6)
-    fluentd (1.7.4)
-      cool.io (>= 1.4.5, < 2.0.0)
-      dig_rb (~> 1.0.0)
-      http_parser.rb (>= 0.5.1, < 0.7.0)
-      msgpack (>= 1.2.0, < 2.0.0)
-      serverengine (>= 2.0.4, < 3.0.0)
-      sigdump (~> 0.2.2)
-      strptime (>= 0.2.2, < 1.0.0)
-      tzinfo (~> 2.0)
-      tzinfo-data (~> 1.0)
-      yajl-ruby (~> 1.0)
-    http_parser.rb (0.6.0)
-    msgpack (1.3.1)
-    parser (2.6.5.0)
-      ast (~> 2.4.0)
-    power_assert (1.1.5)
-    powerpack (0.1.2)
-    rainbow (2.2.2)
-      rake
-    rake (10.5.0)
-    rubocop (0.42.0)
-      parser (>= 2.3.1.1, < 3.0)
-      powerpack (~> 0.1)
-      rainbow (>= 1.99.1, < 3.0)
-      ruby-progressbar (~> 1.7)
-      unicode-display_width (~> 1.0, >= 1.0.1)
-    ruby-progressbar (1.10.1)
-    serverengine (2.1.1)
-      sigdump (~> 0.2.2)
-    sigdump (0.2.4)
-    strptime (0.2.3)
-    test-unit (3.3.4)
-      power_assert
-    tzinfo (2.0.0)
-      concurrent-ruby (~> 1.0)
-    tzinfo-data (1.2019.3)
-      tzinfo (>= 1.0.0)
-    unicode-display_width (1.6.0)
-    yajl-ruby (1.4.1)
-
-PLATFORMS
-  ruby
-
-DEPENDENCIES
-  flexmock (~> 2.0)
-  fluent-plugin-detect-exceptions!
-  rake (~> 10.3)
-  rubocop (= 0.42.0)
-  test-unit (~> 3.0)
-
-BUNDLED WITH
-   2.0.2