fluentd 0.14.17-x86-mingw32 → 1.3.1-x86-mingw32

data/lib/fluent/counter/store.rb

@@ -0,0 +1,205 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/config'
+require 'fluent/counter/error'
+require 'fluent/plugin/storage_local'
+require 'fluent/time'
+
+module Fluent
+  module Counter
+    class Store
+      def self.gen_key(scope, key)
+        "#{scope}\t#{key}"
+      end
+
+      def initialize(opt = {})
+        @log = opt[:log] || $log
+
+        # Notice: This storage is not be implemented auto save.
+        @storage = Plugin.new_storage('local', parent: DummyParent.new(@log))
+        conf = if opt[:path]
+                 {'persistent' => true, 'path' => opt[:path] }
+               else
+                 {'persistent' => false }
+               end
+        @storage.configure(Fluent::Config::Element.new('storage', {}, conf, []))
+      end
+
+      # This class behaves as a configurable plugin for using in storage (OwnedByMixin).
+      class DummyParent
+        include Configurable
+
+        attr_reader :log
+
+        def initialize(log)
+          @log = log
+        end
+
+        def plugin_id
+          'dummy_parent_store'
+        end
+
+        def plugin_id_configured?
+          false
+        end
+
+        # storage_local calls PluginId#plugin_root_dir
+        def plugin_root_dir
+          nil
+        end
+      end
+
+      def start
+        @storage.load
+      end
+
+      def stop
+        @storage.save
+      end
+
+      def init(key, data, ignore: false)
+        ret = if v = get(key)
+                raise InvalidParams.new("#{key} already exists in counter") unless ignore
+                v
+              else
+                @storage.put(key, build_value(data))
+              end
+
+        build_response(ret)
+      end
+
+      def get(key, raise_error: false, raw: false)
+        ret = if raise_error
+                @storage.get(key) or raise UnknownKey.new("`#{key}` doesn't exist in counter")
+              else
+                @storage.get(key)
+              end
+        if raw
+          ret
+        else
+          ret && build_response(ret)
+        end
+      end
+
+      def key?(key)
+        !!@storage.get(key)
+      end
+
+      def delete(key)
+        ret = @storage.delete(key) or raise UnknownKey.new("`#{key}` doesn't exist in counter")
+        build_response(ret)
+      end
+
+      def inc(key, data, force: false)
+        value = data.delete('value')
+        init(key, data) if !key?(key) && force
+        v = get(key, raise_error: true, raw: true)
+        valid_type!(v, value)
+
+        v['total'] += value
+        v['current'] += value
+        t = EventTime.now
+        v['last_modified_at'] = [t.sec, t.nsec]
+        @storage.put(key, v)
+
+        build_response(v)
+      end
+
+      def reset(key)
+        v = get(key, raise_error: true, raw: true)
+        success = false
+        old_data = v.dup
+        now = EventTime.now
+        last_reset_at = EventTime.new(*v['last_reset_at'])
+
+        #  Does it need reset?
+        if (last_reset_at + v['reset_interval']) <= now
+          success = true
+          v['current'] = initial_value(v['type'])
+          t = [now.sec, now.nsec]
+          v['last_reset_at'] = t
+          v['last_modified_at'] = t
+          @storage.put(key, v)
+        end
+
+        {
+          'elapsed_time' => now - last_reset_at,
+          'success' => success,
+          'counter_data' => build_response(old_data)
+        }
+      end
+
+      private
+
+      def build_response(d)
+        {
+          'name' => d['name'],
+          'total' => d['total'],
+          'current' => d['current'],
+          'type' => d['type'],
+          'reset_interval' => d['reset_interval'],
+          'last_reset_at' => EventTime.new(*d['last_reset_at']),
+        }
+      end
+
+      # value is Hash. value requires these fileds.
+      # :name, :total, :current, :type, :reset_interval, :last_reset_at, :last_modified_at
+      def build_value(data)
+        type = data['type'] || 'numeric'
+        now = EventTime.now
+        t = [now.sec, now.nsec]
+
+        v = initial_value(type)
+
+        data.merge(
+          'type' => type,
+          'last_reset_at' => t,
+          'last_modified_at' => t,
+          'current' => v,
+          'total' => v,
+        )
+      end
+
+      def initial_value(type)
+        case type
+        when 'numeric', 'integer' then 0
+        when 'float' then 0.0
+        else raise InvalidParams.new('`type` should be integer, float, or numeric')
+        end
+      end
+
+      def valid_type!(v, value)
+        type = v['type']
+        return unless (type != 'numeric') && (type_str(value) != type)
+        raise InvalidParams.new("`type` is #{type}. You should pass #{type} value as a `value`")
+      end
+
+      def type_str(v)
+        case v
+        when Integer
+          'integer'
+        when Float
+          'float'
+        when Numeric
+          'numeric'
+        else
+          raise InvalidParams.new("`type` should be integer, float, or numeric")
+        end
+      end
+    end
+  end
+end

data/lib/fluent/counter/validator.rb

@@ -0,0 +1,145 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/counter/error'
+
+module Fluent
+  module Counter
+    class Validator
+      VALID_NAME = /\A[a-z][a-zA-Z0-9\-_]*\Z/
+      VALID_SCOPE_NAME = /\A[a-z][\ta-zA-Z0-9\-_]*\Z/
+      VALID_METHODS = %w(establish init delete inc get reset)
+
+      def self.request(data)
+        errors = []
+        raise "Received data is not Hash: #{data}" unless data.is_a?(Hash)
+
+        unless data['id']
+          errors << Fluent::Counter::InvalidRequest.new('Request should include `id`')
+        end
+
+        if !data['method']
+          errors << Fluent::Counter::InvalidRequest.new('Request should include `method`')
+        elsif !(VALID_NAME =~ data['method'])
+          errors << Fluent::Counter::InvalidRequest.new('`method` is the invalid format')
+        elsif !VALID_METHODS.include?(data['method'])
+          errors << Fluent::Counter::MethodNotFound.new("Unknown method name passed: #{data['method']}")
+        end
+
+        errors.map(&:to_hash)
+      end
+
+      def initialize(*types)
+        @types = types.map(&:to_s)
+        @empty = @types.delete('empty')
+      end
+
+      def call(data)
+        success = []
+        errors = []
+
+        if @empty && data.empty?
+          errors << Fluent::Counter::InvalidParams.new('One or more `params` are required')
+        else
+          data.each do |d|
+            begin
+              @types.each { |type| dispatch(type, d) }
+              success << d
+            rescue => e
+              errors << e
+            end
+          end
+        end
+
+        [success, errors]
+      end
+
+      private
+
+      def dispatch(type, data)
+        send("validate_#{type}!", data)
+      rescue NoMethodError => e
+        raise Fluent::Counter::InternalServerError.new(e)
+      end
+    end
+
+    class ArrayValidator < Validator
+      def validate_key!(name)
+        unless name.is_a?(String)
+          raise Fluent::Counter::InvalidParams.new('The type of `key` should be String')
+        end
+
+        unless VALID_NAME =~ name
+          raise Fluent::Counter::InvalidParams.new('`key` is the invalid format')
+        end
+      end
+
+      def validate_scope!(name)
+        unless name.is_a?(String)
+          raise Fluent::Counter::InvalidParams.new('The type of `scope` should be String')
+        end
+
+        unless VALID_SCOPE_NAME =~ name
+          raise Fluent::Counter::InvalidParams.new('`scope` is the invalid format')
+        end
+      end
+    end
+
+    class HashValidator < Validator
+      def validate_name!(hash)
+        name = hash['name']
+        unless name
+          raise Fluent::Counter::InvalidParams.new('`name` is required')
+        end
+
+        unless name.is_a?(String)
+          raise Fluent::Counter::InvalidParams.new('The type of `name` should be String')
+        end
+
+        unless VALID_NAME =~ name
+          raise Fluent::Counter::InvalidParams.new("`name` is the invalid format")
+        end
+      end
+
+      def validate_value!(hash)
+        value = hash['value']
+        unless value
+          raise Fluent::Counter::InvalidParams.new('`value` is required')
+        end
+
+        unless value.is_a?(Numeric)
+          raise Fluent::Counter::InvalidParams.new("The type of `value` type should be Numeric")
+        end
+      end
+
+      def validate_reset_interval!(hash)
+        interval = hash['reset_interval']
+
+        unless interval
+          raise Fluent::Counter::InvalidParams.new('`reset_interval` is required')
+        end
+
+        unless interval.is_a?(Numeric)
+          raise Fluent::Counter::InvalidParams.new('The type of `reset_interval` should be Numeric')
+        end
+
+        if interval < 0
+          raise Fluent::Counter::InvalidParams.new('`reset_interval` should be a positive number')
+        end
+      end
+    end
+  end
+end

data/lib/fluent/env.rb

@@ -20,6 +20,7 @@ module Fluent
   DEFAULT_CONFIG_PATH = ENV['FLUENT_CONF'] || '/etc/fluent/fluent.conf'
   DEFAULT_PLUGIN_DIR = ENV['FLUENT_PLUGIN'] || '/etc/fluent/plugin'
   DEFAULT_SOCKET_PATH = ENV['FLUENT_SOCKET'] || '/var/run/fluent/fluent.sock'
+  DEFAULT_BACKUP_DIR = ENV['FLUENT_BACKUP_DIR'] || '/tmp/fluent'
   DEFAULT_OJ_OPTIONS = {bigdecimal_load: :float, mode: :compat, use_to_json: true}
 
   def self.windows?

data/lib/fluent/event_router.rb

@@ -217,7 +217,7 @@ module Fluent
           @optimizable = if fs_filters.empty?
                            true
                          else
-                           # skip log message when filter is only 1, because its performace is same as non optimized chain.
+                           # skip log message when filter is only 1, because its performance is same as non optimized chain.
                            if @filters.size > 1 && fs_filters.size >= 1
                              $log.info "disable filter chain optimization because #{fs_filters.map(&:class)} uses `#filter_stream` method."
                            end

data/lib/fluent/log.rb

@@ -96,8 +96,12 @@ module Fluent
       @level = logger.level + 1
       @debug_mode = false
       @log_event_enabled = false
-      @time_format = '%Y-%m-%d %H:%M:%S %z '
       @depth_offset = 1
+      @format = nil
+      @time_format = nil
+      @formatter = nil
+
+      self.format = :text
       enable_color out.tty?
       # TODO: This variable name is unclear so we should change to better name.
       @threads_exclude_events = []
@@ -136,12 +140,14 @@ module Fluent
       dl_opts[:log_level] = @level - 1
       logger = ServerEngine::DaemonLogger.new(@out, dl_opts)
       clone = self.class.new(logger, suppress_repeated_stacktrace: @suppress_repeated_stacktrace, process_type: @process_type, worker_id: @worker_id)
+      clone.format = @format
       clone.time_format = @time_format
       clone.log_event_enabled = @log_event_enabled
       # optional headers/attrs are not copied, because new PluginLogger should have another one of it
       clone
     end
 
+    attr_reader :format
     attr_accessor :log_event_enabled
     attr_accessor :out
     attr_accessor :level
@@ -154,6 +160,43 @@ module Fluent
       nil
     end
 
+    def format=(fmt)
+      return if @format == fmt
+
+      @time_format = '%Y-%m-%d %H:%M:%S %z'
+      @time_formatter = Strftime.new(@time_format) rescue nil
+
+      case fmt
+      when :text
+        @format = :text
+        @formatter = Proc.new { |type, time, level, msg|
+          r = caller_line(type, time, @depth_offset, level)
+          r << msg
+          r
+        }
+      when :json
+        @format = :json
+        @formatter = Proc.new { |type, time, level, msg|
+          r = {
+            'time' => format_time(time),
+            'level' => LEVEL_TEXT[level],
+            'message' => msg
+          }
+          if wid = get_worker_id(type)
+            r['worker_id'] = wid
+          end
+          Yajl.dump(r)
+        }
+      end
+
+      nil
+    end
+
+    def time_format=(time_fmt)
+      @time_format = time_fmt
+      @time_formatter = Strftime.new(@time_format) rescue nil
+    end
+
     def reopen!
       # do nothing in @logger.reopen! because it's already reopened in Supervisor.load_config
       @logger.reopen! if @logger
@@ -224,9 +267,9 @@ module Fluent
       end
     end
 
-    def on_trace(&block)
+    def on_trace
       return if @level > LEVEL_TRACE
-      block.call if block
+      yield
     end
 
     def trace(*args, &block)
@@ -235,7 +278,7 @@ module Fluent
       return if skipped_type?(type)
       args << block.call if block
       time, msg = event(:trace, args)
-      puts [@color_trace, caller_line(type, time, @depth_offset, LEVEL_TRACE), msg, @color_reset].join
+      puts [@color_trace, @formatter.call(type, time, LEVEL_TRACE, msg), @color_reset].join
     rescue
       # logger should not raise an exception. This rescue prevents unexpected behaviour.
     end
@@ -245,9 +288,9 @@ module Fluent
       dump_stacktrace(type, backtrace, LEVEL_TRACE)
     end
 
-    def on_debug(&block)
+    def on_debug
       return if @level > LEVEL_DEBUG
-      block.call if block
+      yield
     end
 
     def debug(*args, &block)
@@ -256,7 +299,7 @@ module Fluent
       return if skipped_type?(type)
       args << block.call if block
       time, msg = event(:debug, args)
-      puts [@color_debug, caller_line(type, time, @depth_offset, LEVEL_DEBUG), msg, @color_reset].join
+      puts [@color_debug, @formatter.call(type, time, LEVEL_DEBUG, msg), @color_reset].join
     rescue
     end
     alias DEBUG debug
@@ -265,9 +308,9 @@ module Fluent
       dump_stacktrace(type, backtrace, LEVEL_DEBUG)
     end
 
-    def on_info(&block)
+    def on_info
       return if @level > LEVEL_INFO
-      block.call if block
+      yield
     end
 
     def info(*args, &block)
@@ -276,7 +319,7 @@ module Fluent
       return if skipped_type?(type)
       args << block.call if block
       time, msg = event(:info, args)
-      puts [@color_info, caller_line(type, time, @depth_offset, LEVEL_INFO), msg, @color_reset].join
+      puts [@color_info, @formatter.call(type, time, LEVEL_INFO, msg), @color_reset].join
     rescue
     end
     alias INFO info
@@ -285,9 +328,9 @@ module Fluent
       dump_stacktrace(type, backtrace, LEVEL_INFO)
     end
 
-    def on_warn(&block)
+    def on_warn
       return if @level > LEVEL_WARN
-      block.call if block
+      yield
     end
 
     def warn(*args, &block)
@@ -296,7 +339,7 @@ module Fluent
       return if skipped_type?(type)
       args << block.call if block
       time, msg = event(:warn, args)
-      puts [@color_warn, caller_line(type, time, @depth_offset, LEVEL_WARN), msg, @color_reset].join
+      puts [@color_warn, @formatter.call(type, time, LEVEL_WARN, msg), @color_reset].join
     rescue
     end
     alias WARN warn
@@ -305,9 +348,9 @@ module Fluent
       dump_stacktrace(type, backtrace, LEVEL_WARN)
     end
 
-    def on_error(&block)
+    def on_error
       return if @level > LEVEL_ERROR
-      block.call if block
+      yield
     end
 
     def error(*args, &block)
@@ -316,7 +359,7 @@ module Fluent
       return if skipped_type?(type)
       args << block.call if block
       time, msg = event(:error, args)
-      puts [@color_error, caller_line(type, time, @depth_offset, LEVEL_ERROR), msg, @color_reset].join
+      puts [@color_error, @formatter.call(type, time, LEVEL_ERROR, msg), @color_reset].join
     rescue
     end
     alias ERROR error
@@ -325,9 +368,9 @@ module Fluent
       dump_stacktrace(type, backtrace, LEVEL_ERROR)
     end
 
-    def on_fatal(&block)
+    def on_fatal
       return if @level > LEVEL_FATAL
-      block.call if block
+      yield
     end
 
     def fatal(*args, &block)
@@ -336,7 +379,7 @@ module Fluent
       return if skipped_type?(type)
       args << block.call if block
       time, msg = event(:fatal, args)
-      puts [@color_fatal, caller_line(type, time, @depth_offset, LEVEL_FATAL), msg, @color_reset].join
+      puts [@color_fatal, @formatter.call(type, time, LEVEL_FATAL, msg), @color_reset].join
     rescue
     end
     alias FATAL fatal
@@ -373,19 +416,47 @@ module Fluent
       return if @level > level
 
       time = Time.now
-      line = caller_line(type, time, 5, level)
-      if @suppress_repeated_stacktrace && (Thread.current[:last_repeated_stacktrace] == backtrace)
-        puts ["  ", line, 'suppressed same stacktrace'].join
+
+      if @format == :text
+        line = caller_line(type, time, 5, level)
+        if @suppress_repeated_stacktrace && (Thread.current[:last_repeated_stacktrace] == backtrace)
+          puts ["  ", line, 'suppressed same stacktrace'].join
+        else
+          backtrace.each { |msg|
+            puts ["  ", line, msg].join
+          }
+          Thread.current[:last_repeated_stacktrace] = backtrace if @suppress_repeated_stacktrace
+        end
       else
-        backtrace.each { |msg|
-          puts ["  ", line, msg].join
+        r = {
+          'time' => format_time(time),
+          'level' => LEVEL_TEXT[level],
         }
-        Thread.current[:last_repeated_stacktrace] = backtrace if @suppress_repeated_stacktrace
+        if wid = get_worker_id(type)
+          r['worker_id'] = wid
+        end
+
+        if @suppress_repeated_stacktrace && (Thread.current[:last_repeated_stacktrace] == backtrace)
+          r['message'] = 'suppressed same stacktrace'
+        else
+          r['message'] = backtrace.join("\n")
+          Thread.current[:last_repeated_stacktrace] = backtrace if @suppress_repeated_stacktrace
+        end
+
+        puts Yajl.dump(r)
       end
 
       nil
     end
 
+    def get_worker_id(type)
+      if type == :default && (@process_type == :worker0 || @process_type == :workers)
+        @worker_id
+      else
+        nil
+      end
+    end
+
     def event(level, args)
       time = Time.now
       message = @optional_header ? @optional_header.dup : ''
@@ -426,7 +497,7 @@ module Fluent
                        else
                          "".freeze
                        end
-      log_msg = "#{time.strftime(@time_format)}[#{LEVEL_TEXT[level]}]: #{worker_id_part}"
+      log_msg = "#{format_time(time)} [#{LEVEL_TEXT[level]}]: #{worker_id_part}"
       if @debug_mode
         line = caller(depth+1)[0]
         if match = /^(.+?):(\d+)(?::in `(.*)')?/.match(line)
@@ -438,6 +509,10 @@ module Fluent
       end
       return log_msg
     end
+
+    def format_time(time)
+      @time_formatter ? @time_formatter.exec(time) : time.strftime(@time_format)
+    end
   end
 
 
@@ -450,11 +525,13 @@ module Fluent
     def initialize(logger)
       @logger = logger
       @level = @logger.level
+      @format = nil
       @depth_offset = 2
       if logger.instance_variable_defined?(:@suppress_repeated_stacktrace)
         @suppress_repeated_stacktrace = logger.instance_variable_get(:@suppress_repeated_stacktrace)
       end
 
+      self.format = @logger.format
       enable_color @logger.enable_color?
     end
 
@@ -462,18 +539,24 @@ module Fluent
       @level = Log.str_to_level(log_level_str)
     end
 
+    alias orig_format= format=
     alias orig_enable_color enable_color
 
+    def format=(fmt)
+      self.orig_format = fmt
+      @logger.format = fmt
+    end
+
     def enable_color(b = true)
       orig_enable_color b
       @logger.enable_color b
     end
 
     extend Forwardable
-    def_delegators '@logger', :enable_color?, :enable_debug, :enable_event,
+    def_delegators '@logger', :get_worker_id, :enable_color?, :enable_debug, :enable_event,
       :disable_events, :log_event_enabled, :log_event_enamed=, :time_format, :time_format=,
-      :event, :caller_line, :puts, :write, :<<, :flush, :reset, :out, :out=,
-      :optional_header, :optional_header=, :optional_attrs, :optional_attrs=
+      :time_formatter, :time_formatter=, :event, :caller_line, :puts, :write, :<<, :flush,
+      :reset, :out, :out=, :optional_header, :optional_header=, :optional_attrs, :optional_attrs=
   end
 
 
@@ -542,5 +625,12 @@ module Fluent
         super
       end
     end
+
+    def reopen(path, mode)
+      if mode != 'a'
+        raise "Unsupported mode: #{mode}"
+      end
+      super(path)
+    end
   end
 end