fluentd 0.12.40 → 1.6.2

data/example/out_forward_users.conf

@@ -0,0 +1,65 @@
+<source>
+  @type dummy
+  tag test
+</source>
+<source>
+  @type dummy
+  tag test2
+</source>
+<source>
+  @type dummy
+  tag test3
+</source>
+
+<match test>
+  @type forward
+  flush_interval 0
+  <security>
+    self_hostname output.testing.local
+    shared_key    secure_communication_is_awesome
+  </security>
+  <server>
+    host 127.0.0.1
+    port 24224
+    username user1
+    password yes_this_is_user1
+  </server>
+</match>
+
+<match test2>
+  @type forward
+  flush_interval 0
+  <security>
+    self_hostname output-alt1.testing.local
+    shared_key    secure_communication_is_awesome
+  </security>
+  <server>
+    host 127.0.0.1
+    port 24224
+    username user1
+    password yes_this_is_user1
+  </server>
+  <server>
+    host 127.0.0.1
+    port 24224
+    username user2
+    password yes_this_is_really_user2
+  </server>
+</match>
+
+<match test3>
+  @type forward
+  flush_interval 0
+  <security>
+    self_hostname output-fail.testing.local
+    shared_key    secure_communication_is_awesome
+  </security>
+  <server>
+    host 127.0.0.1
+    port 24224
+    username user3
+    password no_there_are_not_such_user
+    # input plugin warns authentication erro:
+    # [warn]: Authentication failed address="127.0.0.1" hostname="output-fail.testing.local" username="user3"
+  </server>
+</match>

data/example/out_null.conf

@@ -0,0 +1,36 @@
+#
+# bundle exec bin/fluentd -c example/out_buffered_null.conf
+#   (+ --emit-error-log-interval 10)
+<source>
+  @type dummy
+  tag dummy
+  rate 500000000
+  dummy [
+    {"message": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"},
+    {"message": "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"},
+    {"message": "ccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc"}
+  ]
+</source>
+
+<match dummy.**>
+  @type null
+  <buffer>
+    flush_interval 60s
+    chunk_limit_size 1k
+    total_limit_size 4k
+  </buffer>
+</match>
+
+<label error_log>
+  <match **>
+    @type stdout
+    # <buffer>
+    #   flush_interval 1s
+    # </buffer>
+  </match>
+</label>
+
+<match fluent.**>
+  @type relabel
+  @label error_log
+</match>

data/example/secondary_file.conf

@@ -0,0 +1,42 @@
+<system>
+  rpc_endpoint 0.0.0.0:24444
+</system>
+
+<source>
+  @type dummy
+  tag test
+</source>
+
+<source>
+  @type forward
+  @label @raw
+</source>
+
+<label @raw>
+  <match>
+    @type stdout
+  </match>
+</label>
+
+<match test>
+  @type forward
+
+  <buffer time,tag,message>
+    @type memory
+    timekey 2s
+    timekey_wait 1s
+    flush_mode interval
+    flush_interval 1s
+  </buffer>
+
+  <server>
+    host 0.0.0.0
+    port 24224
+  </server>
+
+  <secondary>
+    @type secondary_file
+    directory log/secondary/
+    basename ${tag}_%Y%m%d%L_${message}
+  </secondary>
+</match>

data/example/suppress_config_dump.conf

@@ -0,0 +1,7 @@
+<system>
+  suppress_config_dump false
+</system>
+
+<match data.*>
+  @type stdout
+</match>

data/example/worker_section.conf

@@ -0,0 +1,36 @@
+<system>
+  workers 4
+  root_dir /path/fluentd/root
+</system>
+
+<source> # top-level sections works on all workers in parallel
+  @type forward
+  port 24224
+</source>
+
+<match all> # this sections also works on all workers in parallel
+  @type stdout
+  <inject>
+    worker_id_key worker_id
+  </inject>
+</match>
+
+<worker 0> # this section works only on first worker process
+  <source>
+    @type tail
+    format none
+    path /var/log/fluentd_test.log
+    pos_file /var/log/fluentd_test.pos
+    tag tail
+    rotate_wait 5
+    read_from_head true
+    refresh_interval 60
+  </source>
+
+  <match tail>
+    @type stdout
+    <inject>
+      worker_id_key worker_id
+    </inject>
+  </match>
+</worker>

data/fluent.conf

@@ -31,6 +31,24 @@
 #  tag apache.access
 #</source>
 
+## Mutating event filter
+## Add hostname and tag fields to apache.access tag events
+#<filter apache.access>
+#  @type record_transformer
+#  <record>
+#    hostname ${hostname}
+#    tag ${tag}
+#  </record>
+#</filter>
+
+## Selecting event filter
+## Remove unnecessary events from apache prefixed tag events
+#<filter apache.**>
+#  @type grep
+#  include1 method GET # pass only GET in 'method' field
+#  exclude1 message debug # remove debug event
+#</filter>
+
 # Listen HTTP for monitoring
 # http://localhost:24220/api/plugins
 # http://localhost:24220/api/plugins?type=TYPE
@@ -108,3 +126,14 @@
 #  path /var/log/fluent/else
 #  compress gz
 #</match>
+
+## Label: For handling complex event routing
+#<label @STAGING>
+#  <match system.**>
+#    @type forward
+#    @id staging_forward_output
+#    <server>
+#      host 192.168.0.101
+#    </server>
+#  </match>
+#</label>

data/fluentd.gemspec

@@ -8,28 +8,38 @@ Gem::Specification.new do |gem|
   gem.email         = ["frsyuki@gmail.com"]
   gem.description   = %q{Fluentd is an open source data collector designed to scale and simplify log management. It can collect, process and ship many kinds of data in near real-time.}
   gem.summary       = %q{Fluentd event collector}
-  gem.homepage      = "http://fluentd.org/"
+  gem.homepage      = "https://www.fluentd.org/"
 
   gem.files         = `git ls-files`.split($\)
   gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
   gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
   gem.require_paths = ["lib"]
-  gem.has_rdoc = false
   gem.license = "Apache-2.0"
 
-  gem.required_ruby_version = '>= 1.9.3'
+  gem.required_ruby_version = '>= 2.1'
 
-  gem.add_runtime_dependency("msgpack", [">= 0.5.11", "< 2"])
-  gem.add_runtime_dependency("json", [">= 1.4.3"])
+  gem.add_runtime_dependency("msgpack", [">= 0.7.0", "< 2.0.0"])
   gem.add_runtime_dependency("yajl-ruby", ["~> 1.0"])
-  gem.add_runtime_dependency("cool.io", [">= 1.2.2", "< 2.0.0"])
+  gem.add_runtime_dependency("cool.io", [">= 1.4.5", "< 2.0.0"])
+  gem.add_runtime_dependency("serverengine", [">= 2.0.4", "< 3.0.0"])
   gem.add_runtime_dependency("http_parser.rb", [">= 0.5.1", "< 0.7.0"])
   gem.add_runtime_dependency("sigdump", ["~> 0.2.2"])
-  gem.add_runtime_dependency("tzinfo", [">= 1.0.0"])
-  gem.add_runtime_dependency("tzinfo-data", [">= 1.0.0"])
-  gem.add_runtime_dependency("string-scrub", [">= 0.0.3", "<= 0.0.5"])
+  gem.add_runtime_dependency("tzinfo", ["~> 1.0"])
+  gem.add_runtime_dependency("tzinfo-data", ["~> 1.0"])
+  gem.add_runtime_dependency("strptime", [">= 0.2.2", "< 1.0.0"])
+  gem.add_runtime_dependency("dig_rb", ["~> 1.0.0"])
 
-  gem.add_development_dependency("rake", [">= 0.9.2"])
+  # build gem for a certain platform. see also Rakefile
+  fake_platform = ENV['GEM_BUILD_FAKE_PLATFORM'].to_s
+  gem.platform = fake_platform unless fake_platform.empty?
+  if /mswin|mingw/ =~ fake_platform || (/mswin|mingw/ =~ RUBY_PLATFORM && fake_platform.empty?)
+    gem.add_runtime_dependency("win32-service", ["~> 0.8.3"])
+    gem.add_runtime_dependency("win32-ipc", ["~> 0.6.1"])
+    gem.add_runtime_dependency("win32-event", ["~> 0.6.1"])
+    gem.add_runtime_dependency("windows-pr", ["~> 1.2.5"])
+  end
+
+  gem.add_development_dependency("rake", ["~> 11.0"])
   gem.add_development_dependency("flexmock", ["~> 2.0"])
   gem.add_development_dependency("parallel_tests", ["~> 0.15.3"])
   gem.add_development_dependency("simplecov", ["~> 0.7"])
@@ -37,5 +47,5 @@ Gem::Specification.new do |gem|
   gem.add_development_dependency("timecop", ["~> 0.3"])
   gem.add_development_dependency("test-unit", ["~> 3.2"])
   gem.add_development_dependency("test-unit-rr", ["~> 1.0"])
-  gem.add_development_dependency("oj", ["~> 2.14"])
+  gem.add_development_dependency("oj", [">= 2.14", "< 4"])
 end

data/lib/fluent/agent.rb

@@ -15,9 +15,9 @@
 #
 
 require 'fluent/configurable'
-require 'fluent/engine'
 require 'fluent/plugin'
 require 'fluent/output'
+require 'fluent/match'
 
 module Fluent
   #
@@ -29,16 +29,23 @@ module Fluent
   class Agent
     include Configurable
 
-    def initialize(opts = {})
+    def initialize(log:)
       super()
 
       @context = nil
       @outputs = []
       @filters = []
-      @started_outputs = []
-      @started_filters = []
 
-      @log = Engine.log
+      @lifecycle_control_list = nil
+      # lifecycle_control_list is the list of plugins in this agent, and ordered
+      # from plugins which DOES emit, then DOESN'T emit
+      # (input -> output w/ router -> filter -> output w/o router)
+      # for start: use this order DESC
+      #   (because plugins which appears later in configurations will receive events from plugins which appears earlier)
+      # for stop/before_shutdown/shutdown/after_shutdown/close/terminate: use this order ASC
+      @lifecycle_cache = nil
+
+      @log = log
       @event_router = EventRouter.new(NoMatchMatch.new(log), self)
       @error_collector = nil
     end
@@ -54,9 +61,10 @@ module Fluent
       super
 
       # initialize <match> and <filter> elements
-      conf.elements.select { |e| e.name == 'filter' || e.name == 'match' }.each { |e|
+      conf.elements('filter', 'match').each { |e|
+        next if e.for_another_worker?
         pattern = e.arg.empty? ? '**' : e.arg
-        type = e['@type'] || e['type']
+        type = e['@type']
         raise ConfigError, "Missing '@type' parameter on <#{e.name}> directive" unless type
         if e.name == 'filter'
           add_filter(type, pattern, e)
@@ -66,82 +74,81 @@ module Fluent
       }
     end
 
-    def start
-      @outputs.each { |o|
-        o.start
-        @started_outputs << o
-      }
+    def lifecycle_control_list
+      return @lifecycle_control_list if @lifecycle_control_list
 
-      @filters.each { |f|
-        f.start
-        @started_filters << f
+      lifecycle_control_list = {
+        input: [],
+        output_with_router: [],
+        filter: [],
+        output: [],
       }
-    end
-
-    def shutdown
-      @started_filters.map { |f|
-        Thread.new do
-          begin
-            log.info "shutting down filter#{@context.nil? ? '' : " in #{@context}"}", type: Plugin.lookup_name_from_class(f.class), plugin_id: f.plugin_id
-            f.shutdown
-          rescue => e
-            log.warn "unexpected error while shutting down filter plugins", plugin: f.class, plugin_id: f.plugin_id, error_class: e.class, error: e
-            log.warn_backtrace
-          end
+      if self.respond_to?(:inputs)
+        inputs.each do |i|
+          lifecycle_control_list[:input] << i
         end
-      }.each { |t| t.join }
-
-      # Output plugin as filter emits records at shutdown so emit problem still exist.
-      # This problem will be resolved after actual filter mechanizm.
-      @started_outputs.map { |o|
-        Thread.new do
-          begin
-            log.info "shutting down output#{@context.nil? ? '' : " in #{@context}"}", type: Plugin.lookup_name_from_class(o.class), plugin_id: o.plugin_id
-            o.shutdown
-          rescue => e
-            log.warn "unexpected error while shutting down output plugins", plugin: o.class, plugin_id: o.plugin_id, error_class: e.class, error: e
-            log.warn_backtrace
-          end
+      end
+      outputs.each do |o|
+        if o.has_router?
+          lifecycle_control_list[:output_with_router] << o
+        else
+          lifecycle_control_list[:output] << o
         end
-      }.each { |t| t.join }
-    end
+      end
+      filters.each do |f|
+        lifecycle_control_list[:filter] << f
+      end
 
-    def flush!
-      flush_recursive(@outputs)
+      @lifecycle_control_list = lifecycle_control_list
     end
 
-    def flush_recursive(array)
-      array.each { |o|
-        begin
-          if o.is_a?(BufferedOutput)
-            o.force_flush
-          elsif o.is_a?(MultiOutput)
-            flush_recursive(o.outputs)
-          end
-        rescue => e
-          log.debug "error while force flushing", error_class: e.class, error: e
-          log.debug_backtrace
+    def lifecycle(desc: false)
+      kind_list = if desc
+                    [:output, :filter, :output_with_router]
+                  else
+                    [:output_with_router, :filter, :output]
+                  end
+      kind_list.each do |kind|
+        list = if desc
+                 lifecycle_control_list[kind].reverse
+               else
+                 lifecycle_control_list[kind]
+               end
+        display_kind = (kind == :output_with_router ? :output : kind)
+        list.each do |instance|
+          yield instance, display_kind
         end
-      }
+      end
     end
 
     def add_match(type, pattern, conf)
-      log.info "adding match#{@context.nil? ? '' : " in #{@context}"}", pattern: pattern, type: type
+      log_type = conf.for_this_worker? ? :default : :worker0
+      log.info log_type, "adding match#{@context.nil? ? '' : " in #{@context}"}", pattern: pattern, type: type
 
       output = Plugin.new_output(type)
-      output.router = @event_router
+      output.context_router = @event_router
       output.configure(conf)
       @outputs << output
+      if output.respond_to?(:outputs) && output.respond_to?(:multi_output?) && output.multi_output?
+        # TODO: ruby 2.3 or later: replace `output.respond_to?(:multi_output?) && output.multi_output?` with output&.multi_output?
+        outputs = if output.respond_to?(:static_outputs)
+                    output.static_outputs
+                  else
+                    output.outputs
+                  end
+        @outputs.push(*outputs)
+      end
       @event_router.add_rule(pattern, output)
 
       output
     end
 
     def add_filter(type, pattern, conf)
-      log.info "adding filter#{@context.nil? ? '' : " in #{@context}"}", pattern: pattern, type: type
+      log_type = conf.for_this_worker? ? :default : :worker0
+      log.info log_type, "adding filter#{@context.nil? ? '' : " in #{@context}"}", pattern: pattern, type: type
 
       filter = Plugin.new_filter(type)
-      filter.router = @event_router
+      filter.context_router = @event_router
       filter.configure(conf)
       @filters << filter
       @event_router.add_rule(pattern, filter)
@@ -155,35 +162,5 @@ module Fluent
 
     def handle_emits_error(tag, es, error)
     end
-
-    class NoMatchMatch
-      def initialize(log)
-        @log = log
-        @count = 0
-      end
-
-      def emit(tag, es, chain)
-        # TODO use time instead of num of records
-        c = (@count += 1)
-        if c < 512
-          if Math.log(c) / Math.log(2) % 1.0 == 0
-            @log.warn "no patterns matched", tag: tag
-            return
-          end
-        else
-          if c % 512 == 0
-            @log.warn "no patterns matched", tag: tag
-            return
-          end
-        end
-        @log.on_trace { @log.trace "no patterns matched", tag: tag }
-      end
-
-      def start
-      end
-
-      def shutdown
-      end
-    end
   end
 end

data/lib/fluent/clock.rb

@@ -0,0 +1,62 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+module Fluent
+  module Clock
+    CLOCK_ID = Process::CLOCK_MONOTONIC_RAW rescue Process::CLOCK_MONOTONIC
+
+    @@block_level = 0
+    @@frozen_clock = nil
+
+    def self.now
+      @@frozen_clock || now_raw
+    end
+
+    def self.freeze(dst = nil, &block)
+      return freeze_block(dst, &block) if block_given?
+
+      dst = dst_clock_from_time(dst) if dst.is_a?(Time)
+      @@frozen_clock = dst || now_raw
+    end
+
+    def self.return
+      raise "invalid return while running code in blocks" if @@block_level > 0
+      @@frozen_clock = nil
+    end
+
+    # internal use
+
+    def self.now_raw
+      Process.clock_gettime(CLOCK_ID)
+    end
+
+    def self.dst_clock_from_time(time)
+      diff_sec = Time.now - time
+      now_raw - diff_sec
+    end
+
+    def self.freeze_block(dst)
+      dst = dst_clock_from_time(dst) if dst.is_a?(Time)
+      pre_frozen_clock = @@frozen_clock
+      @@frozen_clock = dst || now_raw
+      @@block_level += 1
+      yield
+    ensure
+      @@block_level -= 1
+      @@frozen_clock = pre_frozen_clock
+    end
+  end
+end