fluentd 0.14.6 → 0.14.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 2a854ece7196a783b2db7b8eb0e0f95b3e6a20eb
-  data.tar.gz: faa92cad1ac199b311d3162a7657d00317981d89
+  metadata.gz: f99bac7f1329f22434907c02e8d382ca2774988e
+  data.tar.gz: fe3cb1733cc27f065397c5135f4f3c579dec81fe
 SHA512:
-  metadata.gz: df3024d66ac13959d17a2ada786eb1da2dcec388901168e8fc740c57161c0fc6e36d4c9d7d87d233cafe8aafc6dfd80e13a2b982d4f80bcc877ee60906367ec8
-  data.tar.gz: de8328984c6438a24c04d395a42f2f2094db93276b2b832dcd9be27f5a3f65be4a2f59f06f764f23d2d1f016ecb87023718846236689ad26abb3fcbea85ed2a4
+  metadata.gz: bc21095fbec80e1ff718ef9179f84081a20d4688b33ee2f7b81821f56501cd5f3645dab77bb6ef98ad175118b5825d1e3ba261ed9baa19ab93214145a515472b
+  data.tar.gz: e2d3978c237c3101ce10146b2afe7ad5f8ffaf180c6d49c5f97cc483e6169ecc2802a2944647b25183cf4dbf81879704a0f61dffab0ac7cdd5b4488269cc30bf

data/ChangeLog

@@ -1,5 +1,51 @@
 # v0.14
 
+## Release v0.14.7 - 2016/10/07
+
+### New features / Enhancement
+
+* in_forward: support to transfer compressed data
+  https://github.com/fluent/fluentd/pull/1179
+* out_stdout: fix to show nanosecond resolution time
+  https://github.com/fluent/fluentd/pull/1249
+* Add option to rotate Fluentd daemon's log
+  https://github.com/fluent/fluentd/pull/1235
+* Add a feature to parse/format numeric time (unix time [+ subsecond value])
+  https://github.com/fluent/fluentd/pull/1254
+* Raise configuration errors for inconsistent <label> configurations
+  https://github.com/fluent/fluentd/pull/1233
+* Fix to instantiate an unconfigured section even for multi: true
+  https://github.com/fluent/fluentd/pull/1210
+* Add validators of placeholders for buffering key extraction
+  https://github.com/fluent/fluentd/pull/1255
+* Fix to show log messages about filter optimization only when needed
+  https://github.com/fluent/fluentd/pull/1227
+* Add some features to write plugins more easily
+  https://github.com/fluent/fluentd/pull/1256
+
+### Bug fixes
+
+* Fix Oj's default option to encode/decode JSON in the same way with Yajl
+  https://github.com/fluent/fluentd/pull/1147
+  https://github.com/fluent/fluentd/pull/1239
+* Fix to raise correct configuration errors
+  https://github.com/fluent/fluentd/pull/1223
+* Fix a bug to call `shutdown` method (and some others) twice
+  https://github.com/fluent/fluentd/pull/1242
+* Fix to enable `chunk.each` only when it's encoded by msgpack
+  https://github.com/fluent/fluentd/pull/1263
+* Fix a bug not to stop enqueue/flush threads correctly
+  https://github.com/fluent/fluentd/pull/1264
+* out_forward: fix a bug that UDP heartbeat doesn't work
+  https://github.com/fluent/fluentd/pull/1238
+* out_file: fix a crash bug when v0.14 enables symlink and resumes existing buffer file chunk generated by v0.12
+  https://github.com/fluent/fluentd/pull/1234
+* in_monitor_agent: fix compatibility problem between outputs of v0.12 and v0.14
+  https://github.com/fluent/fluentd/pull/1232
+* in_tail: fix a bug to crash to read large amount logs
+  https://github.com/fluent/fluentd/pull/1259
+  https://github.com/fluent/fluentd/pull/1261
+
 ## Release v0.14.6 - 2016/09/07
 
 ### Bug fixes

data/bin/fluent-binlog-reader

@@ -0,0 +1,7 @@
+#!/usr/bin/env ruby
+# -*- coding: utf-8 -*-
+here = File.dirname(__FILE__)
+$LOAD_PATH << File.expand_path(File.join(here, '..', 'lib'))
+require 'fluent/command/binlog_reader'
+
+FluentBinlogReader.new.call

data/example/in_dummy_with_compression.conf

@@ -0,0 +1,23 @@
+<source>
+  @type dummy
+  @label @main
+  tag "test.data"
+  size 2
+  rate 10
+  dummy {"message":"yaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaay"}
+  auto_increment_key number
+</source>
+
+<label @main>
+  <match test.data>
+    @type buffered_stdout
+    <buffer>
+      @type file
+      path "#{Dir.pwd}/compressed_buffers"
+      flush_at_shutdown false
+      chunk_limit_size 1m
+      flush_interval 10s
+      compress gzip
+    </buffer>
+  </match>
+</label>

data/lib/fluent/agent.rb

@@ -86,21 +86,12 @@ module Fluent
           lifecycle_control_list[:input] << i
         end
       end
-      recursive_output_traverse = ->(o) {
+      outputs.each do |o|
         if o.has_router?
           lifecycle_control_list[:output_with_router] << o
         else
           lifecycle_control_list[:output] << o
         end
-
-        if o.respond_to?(:outputs)
-          o.outputs.each do |store|
-            recursive_output_traverse.call(store)
-          end
-        end
-      }
-      outputs.each do |o|
-        recursive_output_traverse.call(o)
       end
       filters.each do |f|
         lifecycle_control_list[:filter] << f
@@ -135,9 +126,14 @@ module Fluent
       output.router = @event_router if output.respond_to?(:router=)
       output.configure(conf)
       @outputs << output
-      if output.respond_to?(:outputs) && (output.respond_to?(:multi_output?) && output.multi_output? || output.is_a?(Fluent::MultiOutput))
+      if output.respond_to?(:outputs) && output.respond_to?(:multi_output?) && output.multi_output?
         # TODO: ruby 2.3 or later: replace `output.respond_to?(:multi_output?) && output.multi_output?` with output&.multi_output?
-        @outputs.push(*output.outputs)
+        outputs = if output.respond_to?(:static_outputs)
+                    output.static_outputs
+                  else
+                    output.outputs
+                  end
+        @outputs.push(*outputs)
       end
       @event_router.add_rule(pattern, output)
 

data/lib/fluent/command/binlog_reader.rb

@@ -0,0 +1,234 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'optparse'
+require 'msgpack'
+
+require 'fluent/msgpack_factory'
+require 'fluent/formatter'
+require 'fluent/plugin'
+require 'fluent/config/element'
+
+class FluentBinlogReader
+  SUBCOMMAND = %w(cat head formats)
+  HELP_TEXT = <<HELP
+Usage: fluent-binlog-reader <command> [<args>]
+
+Commands of fluent-binlog-reader:
+   cat     :     Read files sequentially, writing them to standard output.
+   head    :     Display the beginning of a text file.
+   format  :     Display plugins that you can use.
+
+See 'fluent-binlog-reader <command> --help' for more information on a specific command.
+HELP
+
+  def initialize(argv = ARGV)
+    @argv = argv
+  end
+
+  def call
+    command_class = BinlogReaderCommand.const_get(command)
+    command_class.new(@argv).call
+  end
+
+  private
+
+  def command
+    command = @argv.shift
+    if !command || !SUBCOMMAND.include?(command)
+      usage "Required subcommand : #{SUBCOMMAND.join(' | ')}"
+    end
+
+    command.split('_').map(&:capitalize).join('')
+  end
+
+  def usage(msg = nil)
+    puts HELP_TEXT
+    puts "Error: #{msg}" if msg
+    exit 1
+  end
+end
+
+module BinlogReaderCommand
+  class Base
+    def initialize(argv = ARGV)
+      @argv = argv
+
+      @options = { plugin: [] }
+      @opt_parser = OptionParser.new do |opt|
+        opt.separator 'Options:'
+
+        opt.on('-p DIR', '--plugin', 'add library directory path') do |v|
+          @options[:plugin] << v
+        end
+      end
+    end
+
+    def call
+      raise NotImplementedError, 'BUG: command  MUST implement this method'
+    end
+
+    private
+
+    def usage(msg = nil)
+      puts @opt_parser.to_s
+      puts "Error: #{msg}" if msg
+      exit 1
+    end
+
+    def parse_options!
+      @opt_parser.parse!(@argv)
+
+      unless @options[:plugin].empty?
+        if dir = @options[:plugin].find { |d| !Dir.exist?(d) }
+          usage "Directory #{dir} doesn't exist"
+        else
+          @options[:plugin].each do |d|
+            Fluent::Plugin.add_plugin_dir(d)
+          end
+        end
+      end
+    rescue => e
+      usage e
+    end
+  end
+
+  module Formattable
+    DEFAULT_OPTIONS = {
+      format: :out_file
+    }
+
+    def initialize(argv = ARGV)
+      super
+      @options.merge!(DEFAULT_OPTIONS)
+      configure_option_parser
+    end
+
+    private
+
+    def configure_option_parser
+      @options.merge!(config_params: {})
+
+      @opt_parser.banner = "Usage: fluent-binlog-reader #{self.class.to_s.split('::').last.downcase} [options] file"
+
+      @opt_parser.on('-f TYPE', '--format', 'configure output format') do |v|
+        @options[:format] = v.to_sym
+      end
+
+      @opt_parser.on('-e KEY=VALUE', 'configure formatter config params') do |v|
+        key, value = v.split('=')
+        usage "#{v} is invalid. valid format is like `key=value`" unless value
+        @options[:config_params].merge!(key => value)
+      end
+    end
+
+    def lookup_formatter(format, params)
+      conf = Fluent::Config::Element.new('ROOT', '', params, [])
+      formatter = Fluent::Plugin.new_formatter(format)
+
+      if formatter.respond_to?(:configure)
+        formatter.configure(conf)
+      end
+      formatter
+    rescue => e
+      usage e
+    end
+  end
+
+  class Head < Base
+    include Formattable
+
+    DEFAULT_HEAD_OPTIONS = {
+      count: 5
+    }
+
+    def initialize(argv = ARGV)
+      super
+      @options.merge!(default_options)
+      parse_options!
+    end
+
+    def call
+      @formatter = lookup_formatter(@options[:format], @options[:config_params])
+
+      File.open(@path, 'r') do |io|
+        i = 1
+        Fluent::MessagePackFactory.unpacker(io).each do |(time, record)|
+          print @formatter.format(@path, time, record) # path is used for tag
+          break if @options[:count] && i == @options[:count]
+          i += 1
+        end
+      end
+    end
+
+    private
+
+    def default_options
+      DEFAULT_HEAD_OPTIONS
+    end
+
+    def parse_options!
+      @opt_parser.on('-n COUNT', 'Set the number of lines to display') do |v|
+        @options[:count] = v.to_i
+        usage "illegal line count -- #{@options[:count]}" if @options[:count] < 1
+      end
+
+      super
+
+      usage 'Path is required' if @argv.empty?
+      @path = @argv.first
+      usage "#{@path} is not found" unless File.exist?(@path)
+    end
+  end
+
+  class Cat < Head
+    DEFAULT_CAT_OPTIONS = {
+      count: nil                # Overwrite DEFAULT_HEAD_OPTIONS[:count]
+    }
+
+    def default_options
+      DEFAULT_CAT_OPTIONS
+    end
+  end
+
+  class Formats < Base
+    def initialize(argv = ARGV)
+      super
+      parse_options!
+    end
+
+    def call
+      prefix = Fluent::Plugin::FORMATTER_REGISTRY.dir_search_prefix || 'formatter_'
+
+      plugin_dirs = @options[:plugin]
+      unless plugin_dirs.empty?
+        plugin_dirs.each do |d|
+          Dir.glob("#{d}/#{prefix}*.rb").each do |path|
+            require File.absolute_path(path)
+          end
+        end
+      end
+
+      $LOAD_PATH.map do |lp|
+        Dir.glob("#{lp}/#{prefix}*.rb").each do |path|
+          require path
+        end
+      end
+
+      puts Fluent::Plugin::FORMATTER_REGISTRY.map.keys
+    end
+  end
+end

data/lib/fluent/command/fluentd.rb

@@ -81,6 +81,23 @@ op.on('-o', '--log PATH', "log file path") {|s|
   opts[:log_path] = s
 }
 
+ROTATE_AGE = %w(daily weekly monthly)
+op.on('--log-rotate-age AGE', 'generations to keep rotated log files') {|age|
+  if ROTATE_AGE.include?(age)
+    opts[:log_rotate_age] = age
+  else
+    begin
+      opts[:log_rotate_age] = Integer(age)
+    rescue TypeError
+      usage "log-rotate-age should be #{rotate_ages.join(', ')} or a number"
+    end
+  end
+}
+
+op.on('--log-rotate-size BYTES', 'sets the byte size to rotate log files') {|s|
+  opts[:log_rotate_size] = s.to_i
+}
+
 op.on('-i', '--inline-config CONFIG_STRING', "inline config which is appended to the config file on-the-fly") {|s|
   opts[:inline_config] = s
 }
@@ -270,4 +287,3 @@ if opts[:supervise]
 else
   Fluent::Supervisor.new(opts).run_worker
 end
-

data/lib/fluent/compat/file_util.rb

@@ -32,7 +32,7 @@ module Fluent
       end
       module_function :writable?
 
-      # Check file is writable in conjunction wtih mkdir_p(dirname(path))
+      # Check file is writable in conjunction with mkdir_p(dirname(path))
       #
       # @param [String] path File path
       # @return [Boolean] file writable or not

data/lib/fluent/compat/output.rb

@@ -488,7 +488,11 @@ module Fluent
       end
 
       def format_stream(tag, es) # for BufferedOutputTestDriver
-        es.to_msgpack_stream(time_int: @time_as_integer)
+        if @compress == :gzip
+          es.to_compressed_msgpack_stream(time_int: @time_as_integer)
+        else
+          es.to_msgpack_stream(time_int: @time_as_integer)
+        end
       end
 
       def write(chunk)

data/lib/fluent/config/configure_proxy.rb

@@ -208,9 +208,21 @@ module Fluent
         end
       end
 
-      def option_value_type!(name, opts, key, klass)
-        if opts.has_key?(key) && !opts[key].is_a?(klass)
-          raise ArgumentError, "#{name}: #{key} must be a #{klass}, but #{opts[key].class}"
+      def option_value_type!(name, opts, key, klass=nil, type: nil)
+        if opts.has_key?(key)
+          if klass && !opts[key].is_a?(klass)
+            raise ArgumentError, "#{name}: #{key} must be a #{klass}, but #{opts[key].class}"
+          end
+          case type
+          when :boolean
+            unless opts[key].is_a?(TrueClass) || opts[key].is_a?(FalseClass)
+              raise ArgumentError, "#{name}: #{key} must be true or false, but #{opts[key].class}"
+            end
+          when nil
+            # ignore
+          else
+            raise "unknown type: #{type} for option #{key}"
+          end
         end
       end
 
@@ -233,9 +245,10 @@ module Fluent
           raise ArgumentError, "#{name}: unknown config_argument type `#{type}'"
         end
 
+        # options for config_param
         option_value_type!(name, opts, :desc, String)
         option_value_type!(name, opts, :alias, Symbol)
-        option_value_type!(name, opts, :secret, Object) # TrueClass or FalseClass...
+        option_value_type!(name, opts, :secret, type: :boolean)
         option_value_type!(name, opts, :deprecated, String)
         option_value_type!(name, opts, :obsoleted, String)
         if type == :enum
@@ -244,6 +257,7 @@ module Fluent
           end
         end
         option_value_type!(name, opts, :value_type, Symbol) # hash, array
+        option_value_type!(name, opts, :skip_accessor, type: :boolean)
 
         if opts.has_key?(:default)
           config_set_default(name, opts[:default])