fluentd 0.12.10 → 0.12.11

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 1713e75f8506de5f6b0536fad50320386c836d66
-  data.tar.gz: 4b8f4e7728086788c62f2f88b3e45e4ef93d0ccd
+  metadata.gz: f332fd6ebffa5a9f9c318f90e3a14aeca817bd06
+  data.tar.gz: ebf638f6375958a66a285003c4dee6a193fd9914
 SHA512:
-  metadata.gz: 72b4a8dd05fc897004a0313a37fcd2d931723a449dee05b8c6682ceba49ac539cad4dec1cd9ca90fae6563fd1a2893d96de2e7df20248014a5f9da6c416afb03
-  data.tar.gz: c6d11c5e0958a43f8edef4a8ff508687291e02749b850c84f88baae3ef1f78bcf84074718e63b681b0da26decf360d4303063381d894d47eff4ea1f185cfcc81
+  metadata.gz: 6e942e48b3aefbe52818f48827a87efb1f78d0037aaca0ab19612649a2db2e0ab060750ef2c14e36bec96c3ebe9bda3f1eb9d8d8d78701c9e4f3b24ca07e1e65
+  data.tar.gz: 3f1d85a56f16fd280da534cdca74f98cb761c59b77abe9f232b13e677fd774bb0fe97eafed3692ff0b1b78d5b259ba99f5528e5eb68031c41e6f33c97422f10f

data/ChangeLog

@@ -1,5 +1,14 @@
 # v0.12
 
+## Release 0.12.10 - 2015/06/01
+
+### New features / Enhancement
+
+* fluent-cat: Add none format
+  https://github.com/fluent/fluentd/pull/606
+* config: Add secret option
+  https://github.com/fluent/fluentd/pull/604
+
 ## Release 0.12.10 - 2015/05/28
 
 ### New features / Enhancement

data/lib/fluent/command/cat.rb

@@ -28,6 +28,7 @@ socket_path = Fluent::DEFAULT_SOCKET_PATH
 
 config_path = Fluent::DEFAULT_CONFIG_PATH
 format = 'json'
+message_key = 'message'
 
 op.on('-p', '--port PORT', "fluent tcp port (default: #{port})", Integer) {|i|
   port = i
@@ -57,6 +58,14 @@ op.on('--msgpack', "same as: -f msgpack", TrueClass) {|b|
   format = 'msgpack'
 }
 
+op.on('--none', "same as: -f none", TrueClass) {|b|
+  format = 'none'
+}
+
+op.on('--message-key KEY', "key field for none format (default: #{message_key})") {|s|
+  message_key = s
+}
+
 (class<<self;self;end).module_eval do
   define_method(:usage) do |msg|
     puts op.to_s
@@ -290,6 +299,17 @@ when 'msgpack'
     exit 1
   end
 
+when 'none'
+  begin
+    while line = $stdin.gets
+      record = { message_key => line.chomp }
+      w.write(record)
+    end
+  rescue
+    $stderr.puts $!
+    exit 1
+  end
+
 else
   $stderr.puts "Unknown format '#{format}'"
   exit 1

data/lib/fluent/config/element.rb

@@ -29,9 +29,10 @@ module Fluent
         }
         @unused = unused || attrs.keys
         @v1_config = false
+        @corresponding_proxies = [] # some plugins use flat parameters, e.g. in_http doesn't provide <format> section for parser.
       end
 
-      attr_accessor :name, :arg, :elements, :unused, :v1_config
+      attr_accessor :name, :arg, :elements, :unused, :v1_config, :corresponding_proxies
 
       def add_element(name, arg='')
         e = Element.new(name, arg, {}, [])
@@ -103,7 +104,11 @@ module Fluent
           out << "#{indent}<#{@name} #{@arg}>\n"
         end
         each_pair { |k, v|
-          out << "#{nindent}#{k} #{v}\n"
+          if secret_param?(k)
+            out << "#{nindent}#{k} xxxxxx\n"
+          else
+            out << "#{nindent}#{k} #{v}\n"
+          end
         }
         @elements.each { |e|
           out << e.to_s(nest + 1)
@@ -112,6 +117,28 @@ module Fluent
         out
       end
 
+      def to_masked_element
+        element = Element.new(@name, @arg, {}, @elements, @unused)
+        each_pair { |k, v|
+          element[k] = secret_param?(k) ? 'xxxxxx' : v
+        }
+        element
+      end
+
+      def secret_param?(key)
+        return false if @corresponding_proxies.empty?
+
+        param_key = key.to_sym
+        @corresponding_proxies.each { |proxy|
+          block, opts = proxy.params[param_key]
+          if opts && opts.has_key?(:secret)
+            return opts[:secret]
+          end
+        }
+
+        false
+      end
+
       def self.unescape_parameter(v)
         result = ''
         v.each_char { |c| result << LiteralParser.unescape_char(c) }

data/lib/fluent/configurable.rb

@@ -21,8 +21,6 @@ module Fluent
   require 'fluent/registry'
 
   module Configurable
-    attr_reader :config
-
     def self.included(mod)
       mod.extend(ClassMethods)
     end
@@ -47,6 +45,9 @@ module Fluent
 
     def configure(conf)
       @config = conf
+      if class_name = self.class.name # Class.new in tests returns nil so it should be skipped.
+        @config.corresponding_proxies << self.class.configure_proxy(class_name)
+      end
 
       logger = self.respond_to?(:log) ? log : $log
       proxy = self.class.merged_configure_proxy
@@ -64,6 +65,10 @@ module Fluent
       self
     end
 
+    def config
+      @masked_config ||= @config.to_masked_element
+    end
+
     CONFIG_TYPE_REGISTRY = Registry.new(:config_type, 'fluent/plugin/type_')
 
     def self.register_type(type, callable = nil, &block)

data/lib/fluent/engine.rb

@@ -90,12 +90,12 @@ module Fluent
         $log.info "gem '#{spec.name}' version '#{spec.version}'"
       end
 
+      @root_agent.configure(conf)
+      @event_router = @root_agent.event_router
+
       unless @suppress_config_dump
         $log.info "using configuration file: #{conf.to_s.rstrip}"
       end
-
-      @root_agent.configure(conf)
-      @event_router = @root_agent.event_router
     end
 
     def load_plugin_dir(dir)

data/lib/fluent/version.rb

@@ -16,6 +16,6 @@
 
 module Fluent
 
-  VERSION = '0.12.10'
+  VERSION = '0.12.11'
 
 end

data/test/config/test_configurable.rb

@@ -146,6 +146,13 @@ module ConfigurableSpec
       config_param :age, :integer, default: 10
     end
   end
+
+  class Example5
+    include Fluent::Configurable
+
+    config_param :normal_param, :string
+    config_param :secret_param, :string, :secret => true
+  end
 end
 
 module Fluent::Config
@@ -182,18 +189,22 @@ module Fluent::Config
       end
 
       sub_test_case '#configure' do
+        def e(attrs)
+          Fluent::Config::Element.new('', '', attrs, [])
+        end
+
         test 'returns configurable object itself' do
           b2 = ConfigurableSpec::Base2.new
-          assert_instance_of(ConfigurableSpec::Base2, b2.configure({"name1" => "t1", "name5" => "t5", "opt3" => "a"}))
+          assert_instance_of(ConfigurableSpec::Base2, b2.configure(e({"name1" => "t1", "name5" => "t5", "opt3" => "a"})))
         end
 
         test 'raise errors without any specifications for param without defaults' do
           b2 = ConfigurableSpec::Base2.new
-          assert_raise(Fluent::ConfigError) { b2.configure({}) }
-          assert_raise(Fluent::ConfigError) { b2.configure({"name1" => "t1"}) }
-          assert_raise(Fluent::ConfigError) { b2.configure({"name5" => "t5"}) }
-          assert_raise(Fluent::ConfigError) { b2.configure({"name1" => "t1", "name5" => "t5"}) }
-          assert_nothing_raised { b2.configure({"name1" => "t1", "name5" => "t5", "opt3" => "a"}) }
+          assert_raise(Fluent::ConfigError) { b2.configure(e({})) }
+          assert_raise(Fluent::ConfigError) { b2.configure(e({"name1" => "t1"})) }
+          assert_raise(Fluent::ConfigError) { b2.configure(e({"name5" => "t5"})) }
+          assert_raise(Fluent::ConfigError) { b2.configure(e({"name1" => "t1", "name5" => "t5"})) }
+          assert_nothing_raised { b2.configure(e({"name1" => "t1", "name5" => "t5", "opt3" => "a"})) }
 
           assert_equal(["node", false, true, "t1", "base2", "base1", "base2", "t5", "base2"], b2.get_all)
           assert_equal(:a, b2.opt3)
@@ -201,19 +212,19 @@ module Fluent::Config
 
         test 'can configure bool values' do
           b2a = ConfigurableSpec::Base2.new
-          assert_nothing_raised { b2a.configure({"flag1" => "true", "flag2" => "yes", "name1" => "t1", "name5" => "t5", "opt3" => "a"}) }
+          assert_nothing_raised { b2a.configure(e({"flag1" => "true", "flag2" => "yes", "name1" => "t1", "name5" => "t5", "opt3" => "a"})) }
           assert_true(b2a.flag1)
           assert_true(b2a.flag2)
 
           b2b = ConfigurableSpec::Base2.new
-          assert_nothing_raised { b2b.configure({"flag1" => false, "flag2" => "no", "name1" => "t1", "name5" => "t5", "opt3" => "a"}) }
+          assert_nothing_raised { b2b.configure(e({"flag1" => false, "flag2" => "no", "name1" => "t1", "name5" => "t5", "opt3" => "a"})) }
           assert_false(b2b.flag1)
           assert_false(b2b.flag2)
         end
 
         test 'overwrites values of defaults' do
           b2 = ConfigurableSpec::Base2.new
-          b2.configure({"name1" => "t1", "name2" => "t2", "name3" => "t3", "name4" => "t4", "name5" => "t5", "opt1" => "foo", "opt3" => "b"})
+          b2.configure(e({"name1" => "t1", "name2" => "t2", "name3" => "t3", "name4" => "t4", "name5" => "t5", "opt1" => "foo", "opt3" => "b"}))
           assert_equal("t1", b2.name1)
           assert_equal("t2", b2.name2)
           assert_equal("t3", b2.name3)
@@ -227,7 +238,7 @@ module Fluent::Config
         end
 
         test 'enum type rejects values which does not exist in list' do
-          default = {"name1" => "t1", "name2" => "t2", "name3" => "t3", "name4" => "t4", "name5" => "t5", "opt1" => "foo", "opt3" => "b"}
+          default = e({"name1" => "t1", "name2" => "t2", "name3" => "t3", "name4" => "t4", "name5" => "t5", "opt1" => "foo", "opt3" => "b"})
 
           b2 = ConfigurableSpec::Base2.new
           assert_nothing_raised { b2.configure(default) }
@@ -467,12 +478,12 @@ module Fluent::Config
           ex01 = ConfigurableSpec::Example0.new
           assert_raise(Fluent::ConfigError) { ex01.configure(conf0) }
 
-          complete = {
+          complete = e('ROOT', '', {
             "stringvalue" => "s1", "boolvalue" => "yes", "integervalue" => "10",
             "sizevalue" => "10m", "timevalue" => "100s", "floatvalue" => "1.001",
             "hashvalue" => '{"foo":1, "bar":2}',
             "arrayvalue" => '[1,"ichi"]',
-          }
+          })
 
           checker = lambda { |conf| ConfigurableSpec::Example0.new.configure(conf) }
 
@@ -488,12 +499,12 @@ module Fluent::Config
         end
 
         test 'accepts configuration values as string representation' do
-          conf = {
+          conf = e('ROOT', '', {
             "stringvalue" => "s1", "boolvalue" => "yes", "integervalue" => "10",
             "sizevalue" => "10m", "timevalue" => "10m", "floatvalue" => "1.001",
             "hashvalue" => '{"foo":1, "bar":2}',
             "arrayvalue" => '[1,"ichi"]',
-          }
+          })
           ex = ConfigurableSpec::Example0.new.configure(conf)
           assert_equal("s1", ex.stringvalue)
           assert_true(ex.boolvalue)
@@ -506,12 +517,12 @@ module Fluent::Config
         end
 
         test 'accepts configuration values as ruby value representation (especially for DSL)' do
-          conf = {
+          conf = e('ROOT', '', {
             "stringvalue" => "s1", "boolvalue" => true, "integervalue" => 10,
             "sizevalue" => 10 * 1024 * 1024, "timevalue" => 10 * 60, "floatvalue" => 1.001,
             "hashvalue" => {"foo" => 1, "bar" => 2},
             "arrayvalue" => [1,"ichi"],
-          }
+          })
           ex = ConfigurableSpec::Example0.new.configure(conf)
           assert_equal("s1", ex.stringvalue)
           assert_true(ex.boolvalue)
@@ -524,12 +535,12 @@ module Fluent::Config
         end
 
         test 'gets both of true(yes) and false(no) for bool value parameter' do
-          conf = {
+          conf = e('ROOT', '', {
             "stringvalue" => "s1", "integervalue" => 10,
             "sizevalue" => 10 * 1024 * 1024, "timevalue" => 10 * 60, "floatvalue" => 1.001,
             "hashvalue" => {"foo" => 1, "bar" => 2},
             "arrayvalue" => [1,"ichi"],
-          }
+          })
           ex0 = ConfigurableSpec::Example0.new.configure(conf.merge({"boolvalue" => "true"}))
           assert_true(ex0.boolvalue)
 
@@ -673,5 +684,35 @@ module Fluent::Config
         end
       end
     end
+
+    sub_test_case ':secret option' do
+      setup do
+        @conf = Fluent::Config::Element.new('ROOT', '', {'normal_param' => 'param1', 'secret_param' => 'param2'}, [])
+        @example = ConfigurableSpec::Example5.new
+        @example.configure(@conf)
+      end
+
+      test 'to_s hides secret config_param' do
+        @conf.to_s.each_line { |line|
+          key, value = line.strip.split(' ', 2)
+          assert_secret_param(key, value)
+        }
+      end
+
+      test 'config returns masked configuration' do
+        @example.config.each_pair { |key, value|
+          assert_secret_param(key, value)
+        }
+      end
+
+      def assert_secret_param(key, value)
+        case key
+        when 'normal_param'
+          assert_equal 'param1', value
+        when 'secret_param'
+          assert_equal 'xxxxxx', value
+        end
+      end
+    end
   end
 end

data/test/helper.rb

@@ -23,6 +23,18 @@ if ENV['SIMPLE_COV']
   end
 end
 
+# Some tests use Hash instead of Element for configure.
+# We should rewrite these tests in the future and remove this ad-hoc code
+class Hash
+  def corresponding_proxies
+    @corresponding_proxies ||= []
+  end
+
+  def to_masked_element
+    self
+  end
+end
+
 require 'rr'
 require 'test/unit'
 require 'test/unit/rr'

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluentd
 version: !ruby/object:Gem::Version
-  version: 0.12.10
+  version: 0.12.11
 platform: ruby
 authors:
 - Sadayuki Furuhashi
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-05-27 00:00:00.000000000 Z
+date: 2015-06-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: msgpack