fluentd 0.10.62 → 0.12.0.pre.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: cd3088d98e7106ccaf044045d71999404e1357c5
-  data.tar.gz: f232ac836fea58e4719174883df438e2be8c9202
+  metadata.gz: 38b8348d98cec8b31c27e634bd22a9afee293ad0
+  data.tar.gz: 983f2d287d63a26637c95266ce4cd1a85dcd1757
 SHA512:
-  metadata.gz: 55e3054d9f035d210a8c40dca48dd0067749b3d942e8bc6bed63425d8898c11146cca21491d7a32aa614a2d4270dd8ad2f8c7b2e23a06f65ebc0d98b12b53a2e
-  data.tar.gz: eb6293c1d478b3b384badba7024fe68d319797e37c6bebbc12dc7f1cbb89153776cd512b21b9f6ba7f449072906cadbe8907e244be0b76b37eb5bd56e5e1026a
+  metadata.gz: 0b62fb35a7a546cfa741564854091983fdf1d05ff7a85c44cae4dab1889b9f5eb3c5940b32e5739e05bd43a8c7fdbce05723d76a42959fb04cd7b9a6090aee6b
+  data.tar.gz: 8538fff4fcff3ca9fd72ac88681490a7d9e263d53ea51d410b82e486cf6896df7857d158fb6f5dceb55f5ef78072d2bfc3b3bf30faf3dbd00be1b7603452c761

data/.gitignore

@@ -17,6 +17,5 @@ fluent-gem
 fluentd
 pkg/*
 test/tmp/*
-test/config/tmp/*
+spec/config/tmp/*
 make_dist.sh
-Gemfile.local

data/.travis.yml

@@ -14,16 +14,12 @@ os:
 branches:
   only:
     - master
-    - v0.10
 
 gemfile:
   - Gemfile
-  - Gemfile.cool.io.1.1.1
 
 script: bundle exec rake
 
-sudo: false
-
 matrix:
   allow_failures:
     - rvm: ruby-head

data/ChangeLog

@@ -1,75 +1,3 @@
-Release 0.10.62 - 2016/12/15
-
-* Fix gem dependency to fix broken installation
-
-Release 0.10.61 - 2015/03/03
-
-* out_forward: Should handle ECONNREFUSED exeception on TCP heartbeat
-* test: BufferedOutputTestDriver must use correct buffer chunk type
-
-Release 0.10.60 - 2015/02/06
-
-* in_tail: Enable multiline mode if format contains 'multiline'
-* event: Improve EventStream#to_msgpack_stream performance
-* engine: Add tag information emit log
-* Update msgpack dependency to avoid incompatible encoding problem
-
-Release 0.10.59 - 2015/01/23
-
-* in_tail: Support 64bit inode environment in in_tail
-* parser: nginx and apache2 format can now parse access log without http-version
-* buffer: Fix broken calc_retry_wait if Integer is used for retry_wait parameter
-* buffer: Fix to flush a buffer by USR1 signal even on retrying
-* buffer: Fix TimeSlicedOutput doesn't flush with SIGUSR1
-* config: Support @ prefix build-in parameters
-* engine:Fix ThreadError occuring on Signal.trap SIGHUP
-* engine: Fix SIGHUP does not reload config
-* Revert fluent.conf since @type, @id are not available with v0 config
-
-Release 0.10.58 - 2014/12/14
-
-* parser/formatter: Add base class and Plugin.new_xxx/Plugin.register_xxx APIs
-* parser: Fix blank column handling of TSVParser
-* formatter: Add new CSV formatter
-* formatter: Add msgpack format to built-in Formatter to dump records
-* input: Add in_dummy plugin
-* in_http: Add respond_with_empty_img parameter to return empty gif image
-* in_http: Add cors_allow_origins parameter to support CORS request
-* config: Add self.name to configure_proxy error message
-* config: Fix system config using double memory
-* config: Fix v1 config to support multiple tag match
-* config: Fix Config.bool_value regression for nil value
-* buffer: Prevent an exception with large num_retries
-* out_file: Don't create world writable directory in daemon mode
-* add router.emit and router= for compatibility with v0.12
-* Use router instead of Engine.emit
-
-Release 0.10.57 - 2014/11/12
-
-* buffer: Make total_queued_chunk_size thread-safe to avoid race condition with in_monitor_agent
-* Add --no-supervisor option to disable the supervisor for supervision tools like runit
-
-Release 0.10.56 - 2014/10/30
-
-* config: Add Fluent::Config::Section#object_id to avoid NoMethodError by object dump tools
-* mixin: Fix SetTagKeyMixin and SetTimeKeyMixin which ignores config_set_default since 0.10.53
-* input/output: Support 'timezone' configuration parameter (e.g. "+09:00" for JST).
-* log: Use plugin_id instead of object_id in log message for consistency with in_monitor_agent
-
-Release 0.10.55 - 2014/10/17
-
-* config: Relax '@' handling in the plugin configuration for backward compatibility
-
-Release 0.10.54 - 2014/10/17
-
-* config: Change v1's non-quoted string literal behaviour to v0's behaviour
-* config: Add --use-v0-config for keeping old configuration
-* config: Add single quoted literal in v1 configuration
-* config: Improve error message of Array / Hash parse error
-* input: Reduce shutdown time in some network related plugins when use Cool.io 1.2
-* parser: Use ParserError instead of general exception classes
-* Remove unused zfile buffer plugin
-
 Release 0.10.53 - 2014/08/21
 
 * in_tail: Fix forget to detach Closer timer object

data/Gemfile

@@ -1,9 +1,3 @@
 source 'https://rubygems.org/'
 
 gemspec
-
-local_gemfile = File.join(File.dirname(__FILE__), "Gemfile.local")
-if File.exist?(local_gemfile)
-  puts "Loading Gemfile.local ..." if $DEBUG # `ruby -d` or `bundle -v`
-  instance_eval File.read(local_gemfile)
-end

data/Rakefile

@@ -4,8 +4,10 @@ require "bundler/gem_tasks"
 require 'fileutils'
 require 'rake/testtask'
 require 'rake/clean'
+require 'rspec/core'
+require 'rspec/core/rake_task'
 
-task :test => [:base_test]
+task :test => [:base_test, :spec]
 
 desc 'Run test_unit based test'
 Rake::TestTask.new(:base_test) do |t|
@@ -13,7 +15,7 @@ Rake::TestTask.new(:base_test) do |t|
   #  $ bundle exec rake base_test TEST=test/test_specified_path.rb
   #  $ bundle exec rake base_test TEST=test/test_*.rb
   t.libs << "test"
-  t.test_files = Dir["test/**/test_*.rb"].sort
+  t.test_files = (Dir["test/test_*.rb"] + Dir["test/plugin/test_*.rb"] - ["helper.rb"]).sort
   t.verbose = true
   #t.warning = true
 end
@@ -24,7 +26,14 @@ task :parallel_test do
   FileUtils.rm_rf('./test/tmp')
 end
 
-desc 'Run test with simplecov'
+desc 'Run rspec based test'
+RSpec::Core::RakeTask.new(:spec) do |t|
+  t.rspec_opts = %w[-c -f progress -r ./spec/spec_helper.rb]
+  t.pattern = 'spec/**/*_spec.rb'
+  t.verbose = true
+end
+
+desc 'Run rspec with simplecov'
 task :coverage do |t|
   ENV['SIMPLE_COV'] = '1'
   Rake::Task["spec"].invoke

data/example/in_http.conf

@@ -0,0 +1,14 @@
+<source>
+  type http
+  bind 0.0.0.0
+  port 9880
+  body_size_limit 32MB
+  keepalive_timeout 10
+  # backlog 0
+  add_http_headers false
+  format default
+</source>
+
+<match test>
+  type stdout
+</match>

data/example/in_syslog.conf

@@ -0,0 +1,15 @@
+<source>
+  type syslog
+  bind 0.0.0.0
+  port 5140
+  tag test
+  protocol_type udp
+  include_source_host false
+  source_host_key source_host
+  # format ...
+  # with_priority true
+</source>
+
+<match test>
+  type stdout
+</match>

data/example/in_tail.conf

@@ -0,0 +1,14 @@
+<source>
+  type tail
+  format none
+  path /var/log/fluentd_test.log
+  pos_file /var/log/fluentd_test.pos
+  tag test
+  rotate_wait 5
+  read_from_head true
+  refresh_interval 60
+</source>
+
+<match test>
+  type stdout
+</match>

data/example/in_tcp.conf

@@ -0,0 +1,13 @@
+<source>
+  type tcp
+  format none
+  bind 0.0.0.0
+  port 5170
+  delimiter \n
+  source_host_key "host"
+  tag test
+</source>
+
+<match test>
+  type stdout
+</match>

data/example/in_udp.conf

@@ -0,0 +1,13 @@
+<source>
+  type udp
+  format none
+  bind 0.0.0.0
+  port 5160
+  body_size_limit 4KB
+  source_host_key "host"
+  tag test
+</source>
+
+<match test>
+  type stdout
+</match>

data/example/out_copy.conf

@@ -0,0 +1,20 @@
+<source>
+  type forward
+</source>
+
+<match test>
+  type copy
+  deep_copy false
+  <store>
+    type stdout
+  </store>
+  <store>
+    type file
+    path /var/log/fluentd/out_file_test
+    format json
+    buffer_type memory
+    # compress gzip
+    # symlink_path /path/to/symlink
+    append false
+  </store>
+</match>

data/example/out_file.conf

@@ -0,0 +1,13 @@
+<source>
+  type forward
+</source>
+
+<match test>
+  type file
+  path /var/log/fluentd/out_file_test
+  format json
+  buffer_type memory
+  # compress gzip
+  # symlink_path /path/to/symlink
+  append false
+</match>

data/example/out_forward.conf

@@ -0,0 +1,30 @@
+<source>
+  type forward
+</source>
+
+<match test>
+  type forward
+
+  <server>
+    # first server
+    port 24224
+  </server>
+  <server>
+    # second server
+    port 24225
+  </server>
+  <server>
+    # second server
+    port 24226
+    standby
+  </server>
+
+  send_timeout 60
+  heartbeat_type udp
+  heartbeat_interval 1
+  recover_wait 10
+  hard_timeout 60
+  expire_dns_cache nil
+  phi_threshold 16
+  phi_failure_detector true
+</match>

data/fluent.conf

@@ -1,11 +1,8 @@
-# In v1 configuration, type and id are @ prefix parameters.
-# @type and @id are recommended. type and id are still available for backward compatibility
 
 ## built-in TCP input
 ## $ echo <json> | fluent-cat <tag>
 <source>
   type forward
-  id forward_input
 </source>
 
 ## built-in UNIX socket input
@@ -17,8 +14,6 @@
 # http://localhost:8888/<tag>?json=<json>
 <source>
   type http
-  id http_input
-
   port 8888
 </source>
 
@@ -37,20 +32,17 @@
 # http://localhost:24220/api/plugins?tag=MYTAG
 <source>
   type monitor_agent
-  id monitor_agent_input
-
   port 24220
 </source>
 
 # Listen DRb for debug
 <source>
   type debug_agent
-  id debug_agent_input
-
   bind 127.0.0.1
   port 24230
 </source>
 
+
 ## match tag=apache.access and write to file
 #<match apache.access>
 #  type file
@@ -60,14 +52,11 @@
 ## match tag=debug.** and dump to console
 <match debug.**>
   type stdout
-  id stdout_output
 </match>
 
 # match tag=system.** and forward to another fluent server
 <match system.**>
   type forward
-  id forward_output
-
   <server>
     host 192.168.0.11
   </server>
@@ -108,3 +97,4 @@
 #  path /var/log/fluent/else
 #  compress gz
 #</match>
+

data/fluentd.gemspec

@@ -16,23 +16,20 @@ Gem::Specification.new do |gem|
   gem.require_paths = ["lib"]
   gem.has_rdoc = false
 
-  gem.required_ruby_version = '>= 1.9.3'
+  gem.required_ruby_version = '>= 1.9.2'
 
-  gem.add_runtime_dependency("msgpack", [">= 0.4.4", "!= 0.5.0", "!= 0.5.1", "!= 0.5.2", "!= 0.5.3", "!= 0.5.4",
-                                         "!= 0.5.5", "!= 0.5.6", "!= 0.5.7", "!= 0.5.8", "!= 0.5.9", "!= 0.5.10", "< 0.6.0"])
-  gem.add_runtime_dependency("json", [">= 1.4.3", "< 2.0.0"])
+  gem.add_runtime_dependency("msgpack", [">= 0.5.4", "< 0.6.0"])
+  gem.add_runtime_dependency("json", [">= 1.4.3"])
   gem.add_runtime_dependency("yajl-ruby", ["~> 1.0"])
-  gem.add_runtime_dependency("cool.io", [">= 1.1.1", "!= 1.2.0", "< 2.0.0"])
+  gem.add_runtime_dependency("cool.io", [">= 1.2.2", "< 2.0.0"])
   gem.add_runtime_dependency("http_parser.rb", [">= 0.5.1", "< 0.7.0"])
   gem.add_runtime_dependency("sigdump", ["~> 0.2.2"])
-  gem.add_runtime_dependency("tzinfo", [">= 1.0.0"])
-  gem.add_runtime_dependency("tzinfo-data", [">= 1.0.0"])
 
   gem.add_development_dependency("rake", [">= 0.9.2"])
-  gem.add_development_dependency("flexmock", ["< 2.0.0"])
-  gem.add_development_dependency("parallel_tests", [">= 0.15.3", "< 2.0.0"])
+  gem.add_development_dependency("flexmock")
+  gem.add_development_dependency("parallel_tests", [">= 0.15.3"])
+  gem.add_development_dependency("rspec", ["~> 3.0.0"])
   gem.add_development_dependency("simplecov", ["~> 0.6.4"])
-  gem.add_development_dependency("rr", ["~> 1.1.2"])
+  gem.add_development_dependency("rr", [">= 1.0.0"])
   gem.add_development_dependency("timecop", [">= 0.3.0"])
-  gem.add_development_dependency("test-unit", ["~> 3.0.2"])
 end

data/lib/fluent/agent.rb

@@ -0,0 +1,180 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+module Fluent
+  require 'fluent/configurable'
+  require 'fluent/engine'
+
+  #
+  # Agent is a resource unit who manages emittable plugins
+  #
+  # Next step: `fluentd/root_agent.rb`
+  # Next step: `fluentd/label.rb`
+  #
+  class Agent
+    include Configurable
+
+    def initialize(opts = {})
+      # initialize Configurable
+      super()
+
+      @context = nil
+      @outputs = []
+      @filters = []
+      @started_outputs = []
+      @started_filters = []
+
+      @log = Engine.log
+      @event_router = EventRouter.new(self, NoMatchMatch.new(log))
+      @error_collector = nil
+    end
+
+    attr_reader :log
+    attr_reader :outputs
+    attr_reader :context
+    attr_reader :event_router
+    attr_reader :error_collector
+
+    def configure(conf)
+      super
+
+      # initialize <match> and <filter> elements
+      conf.elements.select { |e| e.name == 'filter' || e.name == 'match' }.each { |e|
+        pattern = e.arg.empty? ? '**' : e.arg
+        type = e['type']
+        if e.name == 'filter'
+          add_filter(type, pattern, e)
+        else
+          add_match(type, pattern, e)
+        end
+      }
+    end
+
+    # agent API called by Worker
+    def start
+      @outputs.each { |o|
+        o.start
+        @started_outputs << o
+      }
+
+      @filters.each { |f|
+        f.start
+        @started_filters << f
+      }
+    end
+
+    # agent API called by Worker
+    def shutdown
+      @started_filters.map { |f|
+        Thread.new do
+          begin
+            f.shutdown
+          rescue => e
+            log.warn "unexpected error while shutting down filter plugins", :plugin => f.class, :plugin_id => f.plugin_id, :error_class => e.class, :error => e
+            log.warn_backtrace
+          end
+        end
+      }.each { |t| t.join }
+
+      # Output plugin as filter emits records at shutdown so emit problem still exist.
+      # This problem will be resolved after actual filter mechanizm.
+      @started_outputs.map { |o|
+        Thread.new do
+          begin
+            o.shutdown
+          rescue => e
+            log.warn "unexpected error while shutting down output plugins", :plugin => o.class, :plugin_id => o.plugin_id, :error_class => e.class, :error => e
+            log.warn_backtrace
+          end
+        end
+      }.each { |t| t.join }
+    end
+
+    def flush!
+      flush_recursive(@outputs)
+    end
+
+    def flush_recursive(array)
+      array.each { |o|
+        begin
+          if o.is_a?(BufferedOutput)
+            o.force_flush
+          elsif o.is_a?(MultiOutput)
+            flush_recursive(o.outputs)
+          end
+        rescue => e
+          log.debug "error while force flushing", :error_class => e.class, :error => e
+          log.debug_backtrace
+        end
+      }
+    end
+
+    def add_match(type, pattern, conf)
+      log.info "adding match#{@context.nil? ? '' : " in #{@context}"}", pattern: pattern, type: type
+
+      output = Plugin.new_output(type)
+      output.router = @event_router
+      output.configure(conf)
+      @outputs << output
+      @event_router.add_rule(pattern, output)
+
+      output
+    end
+
+    def add_filter(type, pattern, conf)
+      log.info "adding filter#{@context.nil? ? '' : " in #{@context}"}", pattern: pattern, type: type
+
+      filter = Plugin.new_filter(type)
+      filter.configure(conf)
+      @filters << filter
+      @event_router.add_rule(pattern, filter)
+
+      filter
+    end
+
+    def handle_emits_error(tag, es, e)
+    end
+
+    class NoMatchMatch
+      def initialize(log)
+        @log = log
+        @count = 0
+      end
+
+      def emit(tag, es, chain)
+        # TODO use time instead of num of records
+        c = (@count += 1)
+        if c < 512
+          if Math.log(c) / Math.log(2) % 1.0 == 0
+            @log.warn "no patterns matched", :tag => tag
+            return
+          end
+        else
+          if c % 512 == 0
+            @log.warn "no patterns matched", :tag => tag
+            return
+          end
+        end
+        @log.on_trace { @log.trace "no patterns matched", :tag => tag }
+      end
+
+      def start
+      end
+
+      def shutdown
+      end
+    end
+  end
+end