fluentd 0.14.2 → 0.14.5

data/Rakefile

@@ -7,6 +7,21 @@ require 'rake/clean'
 
 task test: [:base_test]
 
+# 1. update ChangeLog and lib/fluent/version.rb
+# 2. bundle && bundle exec rake build:all
+# 3. release 3 packages built on pkg/ directory
+namespace :build do
+  desc 'Build gems for all platforms'
+  task :all do
+    Bundler.with_clean_env do
+      %w[ruby x86-mingw32 x64-mingw32].each do |name|
+        ENV['GEM_BUILD_FAKE_PLATFORM'] = name
+        Rake::Task["build"].execute
+      end
+    end
+  end
+end
+
 desc 'Run test_unit based test'
 Rake::TestTask.new(:base_test) do |t|
   # To run test for only one file (or file path pattern)

data/appveyor.yml

@@ -28,11 +28,11 @@ environment:
       devkit: C:\Ruby23\DevKit
     - ruby_version: "22-x64"
       devkit: C:\Ruby23-x64\DevKit
+    - ruby_version: "21-x64"
+      devkit: C:\Ruby23-x64\DevKit
     - ruby_version: "22"
       devkit: C:\Ruby23\DevKit
       WIN_RAPID: true
-    - ruby_version: "21-x64"
-      devkit: C:\Ruby23-x64\DevKit
     - ruby_version: "21"
       devkit: C:\Ruby23\DevKit
       WIN_RAPID: true

data/example/in_forward.conf

@@ -8,4 +8,7 @@
 
 <match test>
   @type stdout
+  # <buffer>
+  #   flush_interval 10s
+  # </buffer>
 </match>

data/example/in_forward_client.conf

@@ -0,0 +1,37 @@
+<system>
+  rpc_endpoint 0.0.0.0:24444
+</system>
+
+<source>
+  @type forward
+  port 24224
+  bind 0.0.0.0
+  <security>
+    self_hostname input.testing.local
+    shared_key    secure_communication_is_awesome
+    user_auth     yes
+    allow_anonymous_source no
+    <user>
+      username user1
+      password yes_this_is_user1
+    </user>
+    <user>
+      username user2
+      password yes_this_is_really_user2
+    </user>
+    <user>
+      username user3
+      password noooooo_this_may_not_be_user3
+    </user>
+    <client>
+      # host  127.0.0.1
+      network 127.0.0.0/24
+      shared_key using_different_key_makes_us_secure
+      users user1,user2
+    </client>
+  </security>
+</source>
+
+<match {test,test2,test3,test4}>
+  @type stdout
+</match>

data/example/in_forward_shared_key.conf

@@ -0,0 +1,15 @@
+<system>
+  rpc_endpoint 0.0.0.0:24444
+</system>
+
+<source>
+  @type forward
+  <security>
+    self_hostname input.testing.local
+    shared_key    secure_communication_is_awesome
+  </security>
+</source>
+
+<match test>
+  @type stdout
+</match>

data/example/in_forward_users.conf

@@ -0,0 +1,24 @@
+<system>
+  rpc_endpoint 0.0.0.0:24444
+</system>
+
+<source>
+  @type forward
+  <security>
+    self_hostname input.testing.local
+    shared_key    secure_communication_is_awesome
+    user_auth     yes
+    <user>
+      username user1
+      password yes_this_is_user1
+    </user>
+    <user>
+      username user2
+      password yes_this_is_really_user2
+    </user>
+  </security>
+</source>
+
+<match {test,test2,test3}>
+  @type stdout
+</match>

data/example/multi_filters.conf

@@ -0,0 +1,61 @@
+# This example is to measure optimized filter pipeline performance.
+
+<source>
+  @type dummy
+  tag   test
+  size 1000
+</source>
+
+<filter test>
+  @type grep
+  exclude1 hello .
+</filter>
+
+<filter test>
+  @type grep
+  exclude1 hello .
+</filter>
+
+<filter test>
+  @type grep
+  exclude1 hello .
+</filter>
+
+<filter test>
+  @type grep
+  exclude1 hello .
+</filter>
+
+<filter test>
+  @type grep
+  exclude1 hello .
+</filter>
+
+<filter test>
+  @type grep
+  exclude1 hello .
+</filter>
+
+<filter test>
+  @type grep
+  exclude1 hello .
+</filter>
+
+<filter test>
+  @type grep
+  exclude1 hello .
+</filter>
+
+<filter test>
+  @type grep
+  exclude1 hello .
+</filter>
+
+<filter test>
+  @type grep
+  exclude1 hello .
+</filter>
+
+<match test>
+  @type buffered_null
+</match>

data/example/out_forward.conf

@@ -8,22 +8,22 @@
 
   <server>
     # first server
-    host localhost
+    host 127.0.0.1
     port 24224
   </server>
-  <server>
-    # second server
-    host localhost
-    port 24225
-  </server>
-  <server>
-    # second server
-    host localhost
-    port 24226
-    standby
-  </server>
+  # <server>
+  #   # second server
+  #   host localhost
+  #   port 24225
+  # </server>
+  # <server>
+  #   # second server
+  #   host localhost
+  #   port 24226
+  #   standby
+  # </server>
 
-  flush_interval 1
+  flush_interval 0
   send_timeout 60
   heartbeat_type udp
   heartbeat_interval 1

data/example/out_forward_client.conf

@@ -0,0 +1,109 @@
+<source>
+  @type dummy
+  tag test
+</source>
+<source>
+  @type dummy
+  tag test2
+</source>
+<source>
+  @type dummy
+  tag test3
+</source>
+<source>
+  @type dummy
+  tag test4
+</source>
+<source>
+  @type dummy
+  tag test5
+</source>
+
+<match test>
+  @type forward
+  flush_interval 0
+  <security>
+    self_hostname output.testing.local
+    shared_key    secure_communication_is_awesome
+  </security>
+  <server>
+    host 127.0.0.1
+    port 24224
+    username user1
+    password yes_this_is_user1
+    shared_key using_different_key_makes_us_secure
+  </server>
+</match>
+
+<match test2>
+  @type forward
+  flush_interval 0
+  <security>
+    self_hostname output-alt1.testing.local
+    shared_key    using_different_key_makes_us_secure
+  </security>
+  <server>
+    host 127.0.0.1
+    port 24224
+    username user1
+    password yes_this_is_user1
+  </server>
+  <server>
+    host 127.0.0.1
+    port 24224
+    username user2
+    password yes_this_is_really_user2
+  </server>
+</match>
+
+<match test3>
+  @type forward
+  flush_interval 0
+  <security>
+    self_hostname output-fail1.testing.local
+    # default key: fail
+    shared_key    secure_communication_is_awesome
+  </security>
+  <server>
+    host 127.0.0.1
+    port 24224
+    username user1
+    password yes_this_is_user1
+    # [warn]: Shared key mismatch address="127.0.0.1" hostname="output-fail1.testing.local"
+  </server>
+</match>
+
+<match test4>
+  @type forward
+  flush_interval 0
+  <security>
+    self_hostname output-fail2.testing.local
+    shared_key    using_different_key_makes_us_secure
+  </security>
+  <server>
+    host 127.0.0.1
+    port 24224
+    username user3
+    # user3 (user denied): fail
+    password noooooo_this_may_not_be_user3
+    # [warn]: Authentication failed address="127.0.0.1" hostname="output-fail2.testing.local" username="user3"
+  </server>
+</match>
+
+<match test5>
+  @type forward
+  flush_interval 0
+  <security>
+    self_hostname output-fail3.testing.local
+    shared_key    using_different_key_makes_us_secure
+  </security>
+  <server>
+    # another ip (host rejected): fail
+    # This pattern will work only with Ruby 2.3
+    host "#{Socket.getifaddrs.select{|i| i.addr.ipv4? }.reject{|i| i.addr.ip_address == '127.0.0.1' }.first.addr.ip_address}"
+    port 24224
+    username user1
+    password yes_this_is_user1
+    # [warn]: Anonymous client disallowed address="192.168.1.75" hostname="output-fail3.testing.local"
+  </server>
+</match>

data/example/out_forward_shared_key.conf

@@ -0,0 +1,36 @@
+<source>
+  @type dummy
+  tag test
+</source>
+<source>
+  @type dummy
+  tag test2
+</source>
+
+<match test>
+  @type forward
+  flush_interval 0
+  <security>
+    self_hostname output.testing.local
+    shared_key    secure_communication_is_awesome
+  </security>
+  <server>
+    host 127.0.0.1
+    port 24224
+  </server>
+</match>
+
+<match test2>
+  @type forward
+  flush_interval 0
+  <security>
+    self_hostname output-fail.testing.local
+    shared_key    secure_communication_is_not_awesome
+    # input plugin shows warning for wrong shared_key
+    # 2016-08-08 16:27:00 +0900 [warn]: Shared key mismatch address="127.0.0.1" hostname="output-fail.testing.local"
+  </security>
+  <server>
+    host 127.0.0.1
+    port 24224
+  </server>
+</match>

data/example/out_forward_users.conf

@@ -0,0 +1,65 @@
+<source>
+  @type dummy
+  tag test
+</source>
+<source>
+  @type dummy
+  tag test2
+</source>
+<source>
+  @type dummy
+  tag test3
+</source>
+
+<match test>
+  @type forward
+  flush_interval 0
+  <security>
+    self_hostname output.testing.local
+    shared_key    secure_communication_is_awesome
+  </security>
+  <server>
+    host 127.0.0.1
+    port 24224
+    username user1
+    password yes_this_is_user1
+  </server>
+</match>
+
+<match test2>
+  @type forward
+  flush_interval 0
+  <security>
+    self_hostname output-alt1.testing.local
+    shared_key    secure_communication_is_awesome
+  </security>
+  <server>
+    host 127.0.0.1
+    port 24224
+    username user1
+    password yes_this_is_user1
+  </server>
+  <server>
+    host 127.0.0.1
+    port 24224
+    username user2
+    password yes_this_is_really_user2
+  </server>
+</match>
+
+<match test3>
+  @type forward
+  flush_interval 0
+  <security>
+    self_hostname output-fail.testing.local
+    shared_key    secure_communication_is_awesome
+  </security>
+  <server>
+    host 127.0.0.1
+    port 24224
+    username user3
+    password no_there_are_not_such_user
+    # input plugin warns authentication erro:
+    # [warn]: Authentication failed address="127.0.0.1" hostname="output-fail.testing.local" username="user3"
+  </server>
+</match>

data/example/{out_buffered_null.conf → out_null.conf}

@@ -13,16 +13,20 @@
 </source>
 
 <match dummy.**>
-  @type buffered_null
-  try_flush_interval 60
-  flush_interval 60
-  buffer_chunk_limit 1k
-  buffer_queue_limit 2
+  @type null
+  <buffer>
+    flush_interval 60s
+    chunk_limit_size 1k
+    total_limit_size 4k
+  </buffer>
 </match>
 
 <label error_log>
   <match **>
-    @type stdout # or buffered_stdout
+    @type stdout
+    # <buffer>
+    #   flush_interval 1s
+    # </buffer>
   </match>
 </label>
 

data/example/secondary_file.conf

@@ -0,0 +1,41 @@
+<system>
+  rpc_endpoint 0.0.0.0:24444
+</system>
+
+<source>
+  @type dummy
+  tag test
+</source>
+
+<source>
+  @type forward
+  @label @raw
+</source>
+
+<label @raw>
+  <match>
+    @type stdout
+  </match>
+</label>
+
+<match test>
+  @type forward
+
+  <buffer time,tag,message>
+    type memory
+    timekey 2s
+    timekey_wait 1s
+    flush_interval 1s
+  </buffer>
+
+  <server>
+    host 0.0.0.0
+    port 24224
+  </server>
+
+  <secondary>
+    type secondary_file
+    directory log/secondary/
+    basename ${tag}_%Y%m%d%L_${message}
+  </secondary>
+</match>

data/fluentd.gemspec

@@ -19,30 +19,33 @@ Gem::Specification.new do |gem|
 
   gem.required_ruby_version = '>= 2.1'
 
-  gem.add_runtime_dependency("msgpack", [">= 0.7.0"])
-  gem.add_runtime_dependency("json", [">= 1.4.3"])
+  gem.add_runtime_dependency("msgpack", [">= 0.7.0", "< 2.0.0"])
   gem.add_runtime_dependency("yajl-ruby", ["~> 1.0"])
-  gem.add_runtime_dependency("cool.io", [">= 1.4.3", "< 2.0.0"])
-  gem.add_runtime_dependency("serverengine", [">= 1.6.4"])
+  gem.add_runtime_dependency("cool.io", ["~> 1.4.5"])
+  gem.add_runtime_dependency("serverengine", ["~> 2.0"])
   gem.add_runtime_dependency("http_parser.rb", [">= 0.5.1", "< 0.7.0"])
   gem.add_runtime_dependency("sigdump", ["~> 0.2.2"])
-  gem.add_runtime_dependency("tzinfo", [">= 1.0.0"])
-  gem.add_runtime_dependency("tzinfo-data", [">= 1.0.0"])
-  if /mswin|mingw/ =~ RUBY_PLATFORM
+  gem.add_runtime_dependency("tzinfo", ["~> 1.0"])
+  gem.add_runtime_dependency("tzinfo-data", ["~> 1.0"])
+  gem.add_runtime_dependency("strptime", ["~> 0.1.7"])
+
+  # build gem for a certain platform. see also Rakefile
+  fake_platform = ENV['GEM_BUILD_FAKE_PLATFORM'].to_s
+  gem.platform = fake_platform unless fake_platform.empty?
+  if /mswin|mingw/ =~ fake_platform || (/mswin|mingw/ =~ RUBY_PLATFORM && fake_platform.empty?)
     gem.add_runtime_dependency("win32-service", ["~> 0.8.3"])
     gem.add_runtime_dependency("win32-ipc", ["~> 0.6.1"])
     gem.add_runtime_dependency("win32-event", ["~> 0.6.1"])
     gem.add_runtime_dependency("windows-pr", ["~> 1.2.5"])
   end
-  gem.add_runtime_dependency("strptime", [">= 0.1.7"])
 
-  gem.add_development_dependency("rake", [">= 0.9.2"])
-  gem.add_development_dependency("flexmock", ["~> 2.0.5"])
-  gem.add_development_dependency("parallel_tests", [">= 0.15.3"])
-  gem.add_development_dependency("simplecov", ["~> 0.6.4"])
-  gem.add_development_dependency("rr", [">= 1.0.0"])
-  gem.add_development_dependency("timecop", [">= 0.3.0"])
-  gem.add_development_dependency("test-unit", ["~> 3.1.4"])
-  gem.add_development_dependency("test-unit-rr", ["~> 1.0.3"])
+  gem.add_development_dependency("rake", ["~> 11.0"])
+  gem.add_development_dependency("flexmock", ["~> 2.0"])
+  gem.add_development_dependency("parallel_tests", ["~> 0.15.3"])
+  gem.add_development_dependency("simplecov", ["~> 0.7"])
+  gem.add_development_dependency("rr", ["~> 1.0"])
+  gem.add_development_dependency("timecop", ["~> 0.3"])
+  gem.add_development_dependency("test-unit", ["~> 3.2"])
+  gem.add_development_dependency("test-unit-rr", ["~> 1.0"])
   gem.add_development_dependency("oj", ["~> 2.14"])
 end

data/lib/fluent/agent.rb

@@ -63,6 +63,7 @@ module Fluent
       conf.elements('filter', 'match').each { |e|
         pattern = e.arg.empty? ? '**' : e.arg
         type = e['@type']
+        raise ConfigError, "Missing '@type' parameter on <#{e.name}> directive" unless type
         if e.name == 'filter'
           add_filter(type, pattern, e)
         else
@@ -134,6 +135,10 @@ module Fluent
       output.router = @event_router if output.respond_to?(:router=)
       output.configure(conf)
       @outputs << output
+      if output.respond_to?(:outputs) && (output.respond_to?(:multi_output?) && output.multi_output? || output.is_a?(Fluent::MultiOutput))
+        # TODO: ruby 2.3 or later: replace `output.respond_to?(:multi_output?) && output.multi_output?` with output&.multi_output?
+        @outputs.push(*output.outputs)
+      end
       @event_router.add_rule(pattern, output)
 
       output

data/lib/fluent/command/debug.rb

@@ -63,6 +63,7 @@ end
 require 'fluent/log'
 require 'fluent/engine'
 require 'fluent/system_config'
+require 'serverengine'
 
 include Fluent::SystemConfig::Mixin
 
@@ -90,13 +91,12 @@ include Fluent
 
 puts "Connected to #{uri}."
 puts "Usage:"
-puts "    Engine.match('some.tag').output  : get an output plugin instance"
-puts "    Engine.sources[i]                : get input plugin instances"
-puts "    Plugin.load_plugin(type,name)    : load plugin class (use this if you get DRb::DRbUnknown)"
+puts "    Fluent::Engine.match('some.tag').output  : get an output plugin instance"
+puts "    Fluent::Engine.sources[i]                : get input plugin instances"
+puts "    Fluent::Plugin.load_plugin(type,name)    : load plugin class (use this if you get DRb::DRbUnknown)"
 puts ""
 
 Encoding.default_internal = nil if Encoding.respond_to?(:default_internal)
 
 require 'irb'
 IRB.start
-

data/lib/fluent/compat/handle_tag_and_time_mixin.rb

@@ -14,6 +14,8 @@
 #    limitations under the License.
 #
 
+require 'fluent/time' # TimeFormatter
+
 module Fluent
   module Compat
     module HandleTagAndTimeMixin

data/lib/fluent/compat/output.rb

@@ -19,8 +19,10 @@ require 'fluent/plugin/output'
 require 'fluent/plugin/bare_output'
 require 'fluent/compat/call_super_mixin'
 require 'fluent/compat/formatter_utils'
+require 'fluent/compat/handle_tag_and_time_mixin'
 require 'fluent/compat/parser_utils'
 require 'fluent/compat/propagate_default'
+require 'fluent/compat/record_filter_mixin'
 require 'fluent/compat/output_chain'
 require 'fluent/timezone'
 require 'fluent/mixin'
@@ -265,7 +267,7 @@ module Fluent
           conf.elements << Fluent::Config::Element.new('buffer', '', buf_params, [])
         end
 
-        @includes_record_filter = self.class.ancestors.include?(Fluent::RecordFilterMixin) # TODO rename Compat::RecordFilterMixin
+        @includes_record_filter = self.class.ancestors.include?(Fluent::Compat::RecordFilterMixin)
 
         methods_of_plugin = self.class.instance_methods(false)
         @overrides_emit = methods_of_plugin.include?(:emit)
@@ -361,11 +363,12 @@ module Fluent
         end
 
         meta = metadata(nil, nil, nil)
+        size = es.size
         data = es.map{|time,record| format(tag, time, record) }
         write_guard do
           @buffer.write({meta => data}, enqueue: enqueue)
         end
-        @counters_monitor.synchronize{ @emit_records += es_size }
+        @counters_monitor.synchronize{ @emit_records += size }
         [meta]
       end