fluentd 0.14.11 → 0.14.12

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: ae16df5a5cf19f2500a3c0765117db16915f021f
-  data.tar.gz: b51238e7893315448228735e961520cab6c7fe31
+  metadata.gz: 4c60b140cfb115311fce72ec8be60de12fac24d5
+  data.tar.gz: d7d3eb5f2f0edcdbf3af33dc738f504bb7e9c05b
 SHA512:
-  metadata.gz: 11696eacd069c0ff447da914bc3e8ff824ee6eaae57c2d2d2a3c5080a6dd1252d615affe1053144b6884b960c47bc3bade7798ae58fb8032b94399bad5dfa301
-  data.tar.gz: 92d69b88880c811d42932cd7bf356e12eadbc75550088ffd6f7233537d7fa053f34258bc7d8ebd861e79d23df3a260a5fa0ea489b94b4742b8aec0c906dca631
+  metadata.gz: 4fc6d3c53b3890e58f8193f2e92098ddd2a27e91720f28df6cca793011b792bc91c078b295afa2f73ad36572d7aa6255a2b811002b7070686f32f2837e8445a6
+  data.tar.gz: 373cef8be79781fc5189bf3497e794ea46d4d8218628816539935cb32f4ea44ffa4a3c70cda9533254dff5e0171900cb7d9ebb444e6bdb3fb2d0a8ba42d6a729

data/.travis.yml

@@ -35,11 +35,6 @@ matrix:
   allow_failures:
     - rvm: ruby-head
 
-# no valid version/env for ruby 2.3 right now
-# - rvm: 2.3.0
-#   os: osx
-#   osx_image: ....
-
 branches:
   only:
     - master
@@ -48,6 +43,7 @@ branches:
     - v0.14
 
 sudo: false
+dist: trusty # for TLSv1.2 support
 
 addons:
   apt:

data/ChangeLog

@@ -1,5 +1,57 @@
 # v0.14
 
+## Release v0.14.12 - 2017/01/30
+
+### New features / Enhancements
+* Support multi process workers by `workers` option
+  https://github.com/fluent/fluentd/pull/1386
+* Support TLS transport security layer by server plugin helper, and forward input/output plugins
+  https://github.com/fluent/fluentd/pull/1423
+* Update internal log event handling to route log events to `@FLUENT_LOG` label if configured, suppress log events in startup/shutdown in default
+  https://github.com/fluent/fluentd/pull/1405
+* Rename buffer plugin chunk limit parameters for consistency
+  https://github.com/fluent/fluentd/pull/1412
+* Encode string values from configuration files in UTF8
+  https://github.com/fluent/fluentd/pull/1411
+* Reorder plugin load paths to load rubygem plugins earlier than built-in plugins to overwrite them
+  https://github.com/fluent/fluentd/pull/1410
+* Clock API to control internal thread control
+  https://github.com/fluent/fluentd/pull/1425
+* Validate `config_param` options to restrict unexpected specifications
+  https://github.com/fluent/fluentd/pull/1437
+* formatter: Add `add_newline` option to get formatted lines without newlines
+  https://github.com/fluent/fluentd/pull/1420
+* in_forward: Add `ignore_network_errors_at_startup` option for automated cluster deployment
+  https://github.com/fluent/fluentd/pull/1399
+* in_forward: Close listening socket in #stop, not to accept new connection request in early stage of shutdown
+  https://github.com/fluent/fluentd/pull/1401
+* out_forward: Ensure to pack values in `str` type of msgpack
+  https://github.com/fluent/fluentd/pull/1413
+* in_tail: Add `emit_unmatched_lines` to capture lines which unmatch configured regular expressions
+  https://github.com/fluent/fluentd/pull/1421
+* in_tail: Add `open_on_every_update` to read lines from files opened in exclusive mode on Windows platform
+  https://github.com/fluent/fluentd/pull/1409
+* in_monitor_agent: Add `with_ivars` query parameter to get instance variables only for specified instance variables
+  https://github.com/fluent/fluentd/pull/1393
+* storage_local: Generate file store path using `usage`, with `root_dir` configuration
+  https://github.com/fluent/fluentd/pull/1438
+* Improve test stability
+  https://github.com/fluent/fluentd/pull/1426
+
+### Bug fixes
+* Fix bug to ignore command line options: `--rpc-endpoint`, `--suppress-config-dump`, etc
+  https://github.com/fluent/fluentd/pull/1398
+* Fix bug to block infinitely in shutdown when buffer is full and `overflow_action` is `block`
+  https://github.com/fluent/fluentd/pull/1396
+* buf_file: Fix bug not to use `root_dir` even if configured correctly
+  https://github.com/fluent/fluentd/pull/1417
+* filter_record_transformer: Fix to use BasicObject for clean room
+  https://github.com/fluent/fluentd/pull/1415
+* filter_record_transformer: Fix bug that `remove_keys` doesn't work with `renew_time_key`
+  https://github.com/fluent/fluentd/pull/1433
+* in_monitor_agent: Fix bug to crash with NoMethodError for some output plugins
+  https://github.com/fluent/fluentd/pull/1365
+
 ## Release v0.14.11 - 2016/12/26
 
 ### New features / Enhancements
@@ -21,9 +73,9 @@
 
 ### Bug fixes
 * Fix to set process name of supervisor process
-https://github.com/fluent/fluentd/pull/1380
+  https://github.com/fluent/fluentd/pull/1380
 * in_forward: Fix a bug not to handle "require_ack_response" correctly
-https://github.com/fluent/fluentd/pull/1389
+  https://github.com/fluent/fluentd/pull/1389
 
 
 ## Release v0.14.10 - 2016/12/14

data/example/in_dummy_blocks.conf

@@ -0,0 +1,17 @@
+<source>
+  @type dummy
+  tag dummy
+  rate 100
+  dummy {"message":"yaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaay"}
+</source>
+
+<match dummy>
+  @type null
+  never_flush true
+  <buffer>
+    @type memory
+    overflow_action block
+    chunk_limit_size 1k
+    total_limit_size 2k
+  </buffer>
+</match>

data/example/in_forward_tls.conf

@@ -0,0 +1,14 @@
+<source>
+  @type forward
+  port 24224
+  <transport tls>
+    insecure true
+  </transport>
+</source>
+
+<match test>
+  @type stdout
+  # <buffer>
+  #   flush_interval 10s
+  # </buffer>
+</match>

data/example/in_forward_workers.conf

@@ -0,0 +1,21 @@
+<system>
+  workers 3
+  root_dir "#{File.join(Dir.pwd, 'test', 'tmp', 'root')}"
+</system>
+
+<source>
+  @type forward
+  @id forward_in_1
+</source>
+
+<match test>
+  @type stdout
+  @id stdout_out_1
+  <inject>
+    worker_id_key worker_id
+  </inject>
+  <buffer>
+    @type file
+    flush_interval 1s
+  </buffer>
+</match>

data/example/logevents.conf

@@ -0,0 +1,25 @@
+<source>
+  @type dummy
+  @label @dummylog
+  tag "data"
+  dummy {"message":"yay"}
+</source>
+<label @dummylog>
+  <match **>
+    @type stdout
+  </match>
+</label>
+<label @FLUENT_LOG>
+  <match fluent.debug fluent.info fluent.warn fluent.error fluent.fatal>
+    @type stdout
+    <inject>
+      hostname_key "host"
+    </inject>
+  </match>
+  # <match fluent.{info,warn,error,fatal}>
+  #   @type stdout
+  #   <inject>
+  #     hostname_key "host"
+  #   </inject>
+  # </match>
+</label> 

data/example/out_forward_heartbeat_none.conf

@@ -0,0 +1,16 @@
+<source>
+  @type dummy
+  tag test
+</source>
+
+<match test>
+  @type forward
+  heartbeat_type none
+  <server>
+    host 127.0.0.1
+    port 24224
+  </server>
+  <buffer>
+    flush_mode immediate
+  </buffer>
+</match>

data/example/out_forward_tls.conf

@@ -0,0 +1,18 @@
+<source>
+  @type dummy
+  tag test
+</source>
+
+<match test>
+  @type forward
+  transport tls
+  tls_insecure_mode true
+  <server>
+    # first server
+    host 127.0.0.1
+    port 24224
+  </server>
+  <buffer>
+    flush_interval 0
+  </buffer>
+</match>

data/example/suppress_config_dump.conf

@@ -0,0 +1,7 @@
+<system>
+  suppress_config_dump false
+</system>
+
+<match data.*>
+  @type stdout
+</match>

data/lib/fluent/agent.rb

@@ -17,6 +17,7 @@
 require 'fluent/configurable'
 require 'fluent/plugin'
 require 'fluent/output'
+require 'fluent/match'
 
 module Fluent
   #
@@ -120,7 +121,7 @@ module Fluent
     end
 
     def add_match(type, pattern, conf)
-      log.info "adding match#{@context.nil? ? '' : " in #{@context}"}", pattern: pattern, type: type
+      log.info :worker0, "adding match#{@context.nil? ? '' : " in #{@context}"}", pattern: pattern, type: type
 
       output = Plugin.new_output(type)
       output.context_router = @event_router
@@ -141,7 +142,7 @@ module Fluent
     end
 
     def add_filter(type, pattern, conf)
-      log.info "adding filter#{@context.nil? ? '' : " in #{@context}"}", pattern: pattern, type: type
+      log.info :worker0, "adding filter#{@context.nil? ? '' : " in #{@context}"}", pattern: pattern, type: type
 
       filter = Plugin.new_filter(type)
       filter.context_router = @event_router
@@ -158,35 +159,5 @@ module Fluent
 
     def handle_emits_error(tag, es, error)
     end
-
-    class NoMatchMatch
-      def initialize(log)
-        @log = log
-        @count = 0
-      end
-
-      def emit_events(tag, es)
-        # TODO use time instead of num of records
-        c = (@count += 1)
-        if c < 512
-          if Math.log(c) / Math.log(2) % 1.0 == 0
-            @log.warn "no patterns matched", tag: tag
-            return
-          end
-        else
-          if c % 512 == 0
-            @log.warn "no patterns matched", tag: tag
-            return
-          end
-        end
-        @log.on_trace { @log.trace "no patterns matched", tag: tag }
-      end
-
-      def start
-      end
-
-      def shutdown
-      end
-    end
   end
 end

data/lib/fluent/clock.rb

@@ -0,0 +1,62 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+module Fluent
+  module Clock
+    CLOCK_ID = Process::CLOCK_MONOTONIC_RAW rescue Process::CLOCK_MONOTONIC
+
+    @@block_level = 0
+    @@frozen_clock = nil
+
+    def self.now
+      @@frozen_clock || now_raw
+    end
+
+    def self.freeze(dst = nil, &block)
+      return freeze_block(dst, &block) if block_given?
+
+      dst = dst_clock_from_time(dst) if dst.is_a?(Time)
+      @@frozen_clock = dst || now_raw
+    end
+
+    def self.return
+      raise "invalid return while running code in blocks" if @@block_level > 0
+      @@frozen_clock = nil
+    end
+
+    # internal use
+
+    def self.now_raw
+      Process.clock_gettime(CLOCK_ID)
+    end
+
+    def self.dst_clock_from_time(time)
+      diff_sec = Time.now - time
+      now_raw - diff_sec
+    end
+
+    def self.freeze_block(dst)
+      dst = dst_clock_from_time(dst) if dst.is_a?(Time)
+      pre_frozen_clock = @@frozen_clock
+      @@frozen_clock = dst || now_raw
+      @@block_level += 1
+      yield
+    ensure
+      @@block_level -= 1
+      @@frozen_clock = pre_frozen_clock
+    end
+  end
+end

data/lib/fluent/command/fluentd.rb

@@ -69,6 +69,10 @@ op.on('--no-supervisor', "run fluent worker without supervisor") {
   opts[:standalone_worker] = true
 }
 
+op.on('--workers NUM', "specify the number of workers under supervisor") { |i|
+  opts[:workers] = i.to_i
+}
+
 op.on('--user USER', "change user") {|s|
   opts[:chuser] = s
 }
@@ -98,6 +102,10 @@ op.on('--log-rotate-size BYTES', 'sets the byte size to rotate log files') {|s|
   opts[:log_rotate_size] = s.to_i
 }
 
+op.on('--log-event-verbose', 'enable log events during process startup/shutdown') {|b|
+  opts[:log_event_verbose] = b
+}
+
 op.on('-i', '--inline-config CONFIG_STRING', "inline config which is appended to the config file on-the-fly") {|s|
   opts[:inline_config] = s
 }
@@ -285,5 +293,9 @@ require 'fluent/supervisor'
 if opts[:supervise]
   Fluent::Supervisor.new(opts).run_supervisor
 else
+  if opts[:standalone_worker] && opts[:workers] && opts[:workers] > 1
+    puts "Error: multi workers is not supported with --no-supervisor"
+    exit 2
+  end
   Fluent::Supervisor.new(opts).run_worker
 end

data/lib/fluent/compat/input.rb

@@ -17,7 +17,6 @@
 require 'fluent/plugin'
 require 'fluent/plugin/input'
 require 'fluent/compat/call_super_mixin'
-require 'fluent/process' # to load Fluent::DetachProcessMixin
 
 module Fluent
   module Compat
@@ -44,6 +43,16 @@ module Fluent
       def shutdown
         super
       end
+
+      def detach_process(&block)
+        log.warn "detach_process is not supported in this version. ignored."
+        block.call
+      end
+
+      def detach_multi_process(&block)
+        log.warn "detach_process is not supported in this version. ignored."
+        block.call
+      end
     end
   end
 end

data/lib/fluent/compat/output.rb

@@ -27,7 +27,6 @@ require 'fluent/compat/output_chain'
 require 'fluent/timezone'
 require 'fluent/mixin'
 require 'fluent/event'
-require 'fluent/process' # to load Fluent::DetachProcessMixin
 
 require 'fluent/plugin_helper/compat_parameters'
 
@@ -188,6 +187,16 @@ module Fluent
           end
         end
       end
+
+      def detach_process(&block)
+        log.warn "detach_process is not supported in this version. ignored."
+        block.call
+      end
+
+      def detach_multi_process(&block)
+        log.warn "detach_process is not supported in this version. ignored."
+        block.call
+      end
     end
 
     class MultiOutput < Fluent::Plugin::BareOutput
@@ -398,6 +407,16 @@ module Fluent
           end
         end
       end
+
+      def detach_process(&block)
+        log.warn "detach_process is not supported in this version. ignored."
+        block.call
+      end
+
+      def detach_multi_process(&block)
+        log.warn "detach_process is not supported in this version. ignored."
+        block.call
+      end
     end
 
     class ObjectBufferedOutput < Fluent::Plugin::Output
@@ -525,6 +544,16 @@ module Fluent
           end
         end
       end
+
+      def detach_process(&block)
+        log.warn "detach_process is not supported in this version. ignored."
+        block.call
+      end
+
+      def detach_multi_process(&block)
+        log.warn "detach_process is not supported in this version. ignored."
+        block.call
+      end
     end
 
     class TimeSlicedOutput < Fluent::Plugin::Output
@@ -673,6 +702,16 @@ module Fluent
         end
       end
 
+      def detach_process(&block)
+        log.warn "detach_process is not supported in this version. ignored."
+        block.call
+      end
+
+      def detach_multi_process(&block)
+        log.warn "detach_process is not supported in this version. ignored."
+        block.call
+      end
+
       # Original TimeSlicedOutput#emit doesn't call #format_stream
 
       # #format MUST be implemented in plugin