fluentd 1.0.0.rc1 → 1.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: eadce76b70c3c894fb7a3ccc2a30b02036dbd035
-  data.tar.gz: 2a50d188827fdbd3f4d934c5f5d298cbbde94aa6
+  metadata.gz: 88ca4ec7c1507eb95cad4a86d039a850bfe12424
+  data.tar.gz: b45cd0bea4a3e2091026ab0c0de8da58b2dc7de8
 SHA512:
-  metadata.gz: 361fb845eceae49168b860dea06db95a244e4fcfcd806dc41507db4c7c4a6afd1b76598eba11b8ff81be8bae15e631d36f7c579fadfa5ec51c056174e0f6d51b
-  data.tar.gz: 8908992b6af9c6f8d725162143c2b5c36c7946228e8381928367d6e8fafeab28e069066d48b31f3b9e54f98f32bd8cd3701555b137b5334ca1358ca8db16a86b
+  metadata.gz: 96e8b778f0ce8f7c3c7057a703842499a8b3d492298b54a858b91f8910f68a41618310a1f71f81c7c8628a8c74934a566c55ef31e866aee1c793e02f669a9ddf
+  data.tar.gz: 71131b5600b865b21170ef5a347b74c61ea872146ebfd34985466629c8379ecf6255d5abfc8ebf95a437a862ba285e1b4b8ca6d407fefe3c4703d88f688627f0

data/CHANGELOG.md

@@ -1,9 +1,13 @@
 # v1.0
 
-## Release v1.0.0.rc1 - 2017/11/29
+## Release v1.0.0 - 2017/12/6
 
 ### New features / Enhancements
 
+* out_copy: Support ignore_error argument in `<store>`
+  https://github.com/fluent/fluentd/pull/1764
+* server helper: Improve resource usage of TLS transport
+  https://github.com/fluent/fluentd/pull/1764
 * Disable tracepoint feature to omit unnecessary insts
   https://github.com/fluent/fluentd/pull/1764
 
@@ -13,9 +17,33 @@
   https://github.com/fluent/fluentd/pull/1686
 * plugin: Combine before_shutdown and shutdown call in one sequence.
   https://github.com/fluent/fluentd/pull/1763
+* Add description to parsers
+  https://github.com/fluent/fluentd/pull/1776
+  https://github.com/fluent/fluentd/pull/1777
+  https://github.com/fluent/fluentd/pull/1778
+  https://github.com/fluent/fluentd/pull/1779
+  https://github.com/fluent/fluentd/pull/1780
+* filter_parser: Add parameter description
+  https://github.com/fluent/fluentd/pull/1773
+* plugin: Combine before_shutdown and shutdown call in one sequence.
+  https://github.com/fluent/fluentd/pull/1763
 
 # v0.14
 
+## Release v0.14.25 - 2017/11/29
+
+### New features / Enhancements
+
+* Disable tracepoint feature to omit unnecessary insts
+  https://github.com/fluent/fluentd/pull/1764
+
+### Bug fixes
+
+* out_forward: Don't update retry state when failed to get ack response.
+  https://github.com/fluent/fluentd/pull/1686
+* plugin: Combine before_shutdown and shutdown call in one sequence.
+  https://github.com/fluent/fluentd/pull/1763
+
 ## Release v0.14.24 - 2017/11/24
 
 ### New features / Enhancements

data/lib/fluent/plugin/filter_parser.rb

@@ -26,12 +26,19 @@ module Fluent::Plugin
 
     helpers :parser, :record_accessor, :compat_parameters
 
+    desc 'Specify field name in the record to parse.'
     config_param :key_name, :string
+    desc 'Keep original key-value pair in parsed result.'
     config_param :reserve_data, :bool, default: false
+    desc 'Keep original event time in parsed result.'
     config_param :reserve_time, :bool, default: false
+    desc 'Store parsed values with specified key name prefix.'
     config_param :inject_key_prefix, :string, default: nil
+    desc 'If true, invalid string is replaced with safe characters and re-parse it.'
     config_param :replace_invalid_sequence, :bool, default: false
+    desc 'Store parsed values as a hash value in a field.'
     config_param :hash_value_field, :string, default: nil
+    desc 'Emit invalid record to @ERROR label'
     config_param :emit_invalid_record_to_error, :bool, default: true
 
     attr_reader :parser

data/lib/fluent/plugin/multi_output.rb

@@ -29,6 +29,7 @@ module Fluent
       helpers :event_emitter # to get router from agent, which will be supplied to child plugins
 
       config_section :store, param_name: :stores, multi: true, required: true do
+        config_argument :arg, :string, default: ''
         config_param :@type, :string, default: nil
       end
 

data/lib/fluent/plugin/out_copy.rb

@@ -25,6 +25,21 @@ module Fluent::Plugin
     desc 'If true, pass different record to each `store` plugin.'
     config_param :deep_copy, :bool, default: false
 
+    attr_reader :ignore_errors
+
+    def initialize
+      super
+      @ignore_errors = []
+    end
+
+    def configure(conf)
+      super
+
+      @stores.each { |store|
+        @ignore_errors << (store.arg == 'ignore_error')
+      }
+    end
+
     def multi_workers_ready?
       true
     end
@@ -38,8 +53,16 @@ module Fluent::Plugin
         es = m
       end
 
-      outputs.each do |output|
-        output.emit_events(tag, @deep_copy ? es.dup : es)
+      outputs.each_with_index do |output, i|
+        begin
+          output.emit_events(tag, @deep_copy ? es.dup : es)
+        rescue => e
+          if @ignore_errors[i]
+            log.error "ignore emit error", error: e
+          else
+            raise e
+          end
+        end
       end
     end
   end

data/lib/fluent/plugin/parser_json.rb

@@ -27,6 +27,7 @@ module Fluent
       Plugin.register_parser('json', self)
 
       config_set_default :time_key, 'time'
+      desc 'Set JSON parser'
       config_param :json_parser, :enum, list: [:oj, :yajl, :json], default: :oj
 
       config_set_default :time_type, :float

data/lib/fluent/plugin/parser_multiline.rb

@@ -22,6 +22,7 @@ module Fluent
     class MultilineParser < Parser
       Plugin.register_parser('multiline', self)
 
+      desc 'Specify regexp pattern for start line of multiple lines'
       config_param :format_firstline, :string, default: nil
 
       FORMAT_MAX_NUM = 20

data/lib/fluent/plugin/parser_none.rb

@@ -23,6 +23,7 @@ module Fluent
     class NoneParser < Parser
       Plugin.register_parser('none', self)
 
+      desc 'Field name to contain logs'
       config_param :message_key, :string, default: 'message'
 
       def parse(text)

data/lib/fluent/plugin/parser_regexp.rb

@@ -21,8 +21,11 @@ module Fluent
     class RegexpParser < Parser
       Plugin.register_parser("regexp", self)
 
+      desc 'Regular expression for matching logs'
       config_param :expression, :string
+      desc 'Ignore case in matching'
       config_param :ignorecase, :bool, default: false
+      desc 'Build regular expression as a multline mode'
       config_param :multiline, :bool, default: false
 
       config_set_default :time_key, 'time'

data/lib/fluent/plugin/parser_syslog.rb

@@ -32,8 +32,11 @@ module Fluent
       REGEXP_DETECT_RFC5424 = /^\<.*\>[1-9]\d{0,2}/
 
       config_set_default :time_format, "%b %d %H:%M:%S"
+      desc 'If the incoming logs have priority prefix, e.g. <9>, set true'
       config_param :with_priority, :bool, default: false
+      desc 'Specify protocol format'
       config_param :message_format, :enum, list: [:rfc3164, :rfc5424, :auto], default: :rfc3164
+      desc 'Specify time format for event time for rfc5424 protocol'
       config_param :rfc5424_time_format, :string, default: "%Y-%m-%dT%H:%M:%S.%L%z"
 
       def initialize

data/lib/fluent/plugin_helper/server.rb

@@ -661,22 +661,39 @@ module Fluent
             end
           end
 
+          if RUBY_VERSION.to_f >= 2.3
+            NONBLOCK_ARG = {exception: false}
+            def try_handshake
+              @_handler_socket.accept_nonblock(NONBLOCK_ARG)
+            end
+          else
+            def try_handshake
+              @_handler_socket.accept_nonblock
+            rescue IO::WaitReadable
+              :wait_readable
+            rescue IO::WaitWritable
+              :wait_writable
+            end
+          end
+
           def try_tls_accept
             return true if @_handler_accepted
 
             begin
-              @_handler_socket.accept_nonblock # this method call actually try to do handshake via TLS
-              @_handler_accepted = true
-
-              @callback_connection = TLSCallbackSocket.new(self)
-              @connect_callback.call(@callback_connection)
-              unless @data_callback
-                raise "connection callback must call #data to set data callback"
+              result = try_handshake # this method call actually try to do handshake via TLS
+              if result == :wait_readable || result == :wait_writable
+                # retry accept_nonblock: there aren't enough data in underlying socket buffer
+              else
+                @_handler_accepted = true
+
+                @callback_connection = TLSCallbackSocket.new(self)
+                @connect_callback.call(@callback_connection)
+                unless @data_callback
+                  raise "connection callback must call #data to set data callback"
+                end
+
+                return true
               end
-              return true
-
-            rescue IO::WaitReadable, IO::WaitWritable
-              # retry accept_nonblock: there aren't enough data in underlying socket buffer
             rescue OpenSSL::SSL::SSLError => e
               @log.trace "unexpected error before accepting TLS connection", error: e
               close rescue nil
@@ -700,6 +717,7 @@ module Fluent
           def on_writable
             begin
               @mutex.synchronize do
+                # Consider write_nonblock with {exception: false} when IO::WaitWritable error happens frequently.
                 written_bytes = @_handler_socket.write_nonblock(@_handler_write_buffer)
                 @_handler_write_buffer.slice!(0, written_bytes)
                 super

data/lib/fluent/version.rb

@@ -16,6 +16,6 @@
 
 module Fluent
 
-  VERSION = '1.0.0.rc1'
+  VERSION = '1.0.0'
 
 end

data/test/plugin/test_out_copy.rb

@@ -156,5 +156,36 @@ class CopyOutputTest < Test::Unit::TestCase
       end
     end
   end
+
+  IGNORE_ERROR_CONFIG = %[
+    <store ignore_error>
+      @type test
+      name c0
+    </store>
+    <store ignore_error>
+      @type test
+      name c1
+    </store>
+    <store>
+      @type test
+      name c2
+    </store>
+  ]
+
+  def test_ignore_error
+    d = create_driver(IGNORE_ERROR_CONFIG)
+
+    # override to raise an error
+    d.instance.outputs[0].define_singleton_method(:process) do |tag, es|
+      raise ArgumentError, 'Failed'
+    end
+
+    time = Time.parse("2011-01-02 13:14:15 UTC").to_i
+    assert_nothing_raised do
+      d.run(default_tag: 'test') do
+        d.feed(time, {"a"=>1})
+      end
+    end
+  end
 end
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluentd
 version: !ruby/object:Gem::Version
-  version: 1.0.0.rc1
+  version: 1.0.0
 platform: ruby
 authors:
 - Sadayuki Furuhashi
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-11-29 00:00:00.000000000 Z
+date: 2017-12-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: msgpack
@@ -737,9 +737,9 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '2.1'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">"
+  - - ">="
     - !ruby/object:Gem::Version
-      version: 1.3.1
+      version: '0'
 requirements: []
 rubyforge_project: 
 rubygems_version: 2.6.13