fluentd 1.2.0.pre1 → 1.2.0

data/lib/fluent/counter/store.rb

@@ -0,0 +1,205 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/config'
+require 'fluent/counter/error'
+require 'fluent/plugin/storage_local'
+require 'fluent/time'
+
+module Fluent
+  module Counter
+    class Store
+      def self.gen_key(scope, key)
+        "#{scope}\t#{key}"
+      end
+
+      def initialize(opt = {})
+        @log = opt[:log] || $log
+
+        # Notice: This storage is not be implemented auto save.
+        @storage = Plugin.new_storage('local', parent: DummyParent.new(@log))
+        conf = if opt[:path]
+                 {'persistent' => true, 'path' => opt[:path] }
+               else
+                 {'persistent' => false }
+               end
+        @storage.configure(Fluent::Config::Element.new('storage', {}, conf, []))
+      end
+
+      # This class behaves as a configurable plugin for using in storage (OwnedByMixin).
+      class DummyParent
+        include Configurable
+
+        attr_reader :log
+
+        def initialize(log)
+          @log = log
+        end
+
+        def plugin_id
+          'dummy_parent_store'
+        end
+
+        def plugin_id_configured?
+          false
+        end
+
+        # storage_local calls PluginId#plugin_root_dir
+        def plugin_root_dir
+          nil
+        end
+      end
+
+      def start
+        @storage.load
+      end
+
+      def stop
+        @storage.save
+      end
+
+      def init(key, data, ignore: false)
+        ret = if v = get(key)
+                raise InvalidParams.new("#{key} already exists in counter") unless ignore
+                v
+              else
+                @storage.put(key, build_value(data))
+              end
+
+        build_response(ret)
+      end
+
+      def get(key, raise_error: false, raw: false)
+        ret = if raise_error
+                @storage.get(key) or raise UnknownKey.new("`#{key}` doesn't exist in counter")
+              else
+                @storage.get(key)
+              end
+        if raw
+          ret
+        else
+          ret && build_response(ret)
+        end
+      end
+
+      def key?(key)
+        !!@storage.get(key)
+      end
+
+      def delete(key)
+        ret = @storage.delete(key) or raise UnknownKey.new("`#{key}` doesn't exist in counter")
+        build_response(ret)
+      end
+
+      def inc(key, data, force: false)
+        value = data.delete('value')
+        init(key, data) if !key?(key) && force
+        v = get(key, raise_error: true, raw: true)
+        valid_type!(v, value)
+
+        v['total'] += value
+        v['current'] += value
+        t = EventTime.now
+        v['last_modified_at'] = [t.sec, t.nsec]
+        @storage.put(key, v)
+
+        build_response(v)
+      end
+
+      def reset(key)
+        v = get(key, raise_error: true, raw: true)
+        success = false
+        old_data = v.dup
+        now = EventTime.now
+        last_reset_at = EventTime.new(*v['last_reset_at'])
+
+        #  Does it need reset?
+        if (last_reset_at + v['reset_interval']) <= now
+          success = true
+          v['current'] = initial_value(v['type'])
+          t = [now.sec, now.nsec]
+          v['last_reset_at'] = t
+          v['last_modified_at'] = t
+          @storage.put(key, v)
+        end
+
+        {
+          'elapsed_time' => now - last_reset_at,
+          'success' => success,
+          'counter_data' => build_response(old_data)
+        }
+      end
+
+      private
+
+      def build_response(d)
+        {
+          'name' => d['name'],
+          'total' => d['total'],
+          'current' => d['current'],
+          'type' => d['type'],
+          'reset_interval' => d['reset_interval'],
+          'last_reset_at' => EventTime.new(*d['last_reset_at']),
+        }
+      end
+
+      # value is Hash. value requires these fileds.
+      # :name, :total, :current, :type, :reset_interval, :last_reset_at, :last_modified_at
+      def build_value(data)
+        type = data['type'] || 'numeric'
+        now = EventTime.now
+        t = [now.sec, now.nsec]
+
+        v = initial_value(type)
+
+        data.merge(
+          'type' => type,
+          'last_reset_at' => t,
+          'last_modified_at' => t,
+          'current' => v,
+          'total' => v,
+        )
+      end
+
+      def initial_value(type)
+        case type
+        when 'numeric', 'integer' then 0
+        when 'float' then 0.0
+        else raise InvalidParams.new('`type` should be integer, float, or numeric')
+        end
+      end
+
+      def valid_type!(v, value)
+        type = v['type']
+        return unless (type != 'numeric') && (type_str(value) != type)
+        raise InvalidParams.new("`type` is #{type}. You should pass #{type} value as a `value`")
+      end
+
+      def type_str(v)
+        case v
+        when Integer
+          'integer'
+        when Float
+          'float'
+        when Numeric
+          'numeric'
+        else
+          raise InvalidParams.new("`type` should be integer, float, or numeric")
+        end
+      end
+    end
+  end
+end

data/lib/fluent/counter/validator.rb

@@ -0,0 +1,145 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/counter/error'
+
+module Fluent
+  module Counter
+    class Validator
+      VALID_NAME = /\A[a-z][a-zA-Z0-9\-_]*\Z/
+      VALID_SCOPE_NAME = /\A[a-z][\ta-zA-Z0-9\-_]*\Z/
+      VALID_METHODS = %w(establish init delete inc get reset)
+
+      def self.request(data)
+        errors = []
+        raise "Received data is not Hash: #{data}" unless data.is_a?(Hash)
+
+        unless data['id']
+          errors << Fluent::Counter::InvalidRequest.new('Request should include `id`')
+        end
+
+        if !data['method']
+          errors << Fluent::Counter::InvalidRequest.new('Request should include `method`')
+        elsif !(VALID_NAME =~ data['method'])
+          errors << Fluent::Counter::InvalidRequest.new('`method` is the invalid format')
+        elsif !VALID_METHODS.include?(data['method'])
+          errors << Fluent::Counter::MethodNotFound.new("Unknown method name passed: #{data['method']}")
+        end
+
+        errors.map(&:to_hash)
+      end
+
+      def initialize(*types)
+        @types = types.map(&:to_s)
+        @empty = @types.delete('empty')
+      end
+
+      def call(data)
+        success = []
+        errors = []
+
+        if @empty && data.empty?
+          errors << Fluent::Counter::InvalidParams.new('One or more `params` are required')
+        else
+          data.each do |d|
+            begin
+              @types.each { |type| dispatch(type, d) }
+              success << d
+            rescue => e
+              errors << e
+            end
+          end
+        end
+
+        [success, errors]
+      end
+
+      private
+
+      def dispatch(type, data)
+        send("validate_#{type}!", data)
+      rescue NoMethodError => e
+        raise Fluent::Counter::InternalServerError.new(e)
+      end
+    end
+
+    class ArrayValidator < Validator
+      def validate_key!(name)
+        unless name.is_a?(String)
+          raise Fluent::Counter::InvalidParams.new('The type of `key` should be String')
+        end
+
+        unless VALID_NAME =~ name
+          raise Fluent::Counter::InvalidParams.new('`key` is the invalid format')
+        end
+      end
+
+      def validate_scope!(name)
+        unless name.is_a?(String)
+          raise Fluent::Counter::InvalidParams.new('The type of `scope` should be String')
+        end
+
+        unless VALID_SCOPE_NAME =~ name
+          raise Fluent::Counter::InvalidParams.new('`scope` is the invalid format')
+        end
+      end
+    end
+
+    class HashValidator < Validator
+      def validate_name!(hash)
+        name = hash['name']
+        unless name
+          raise Fluent::Counter::InvalidParams.new('`name` is required')
+        end
+
+        unless name.is_a?(String)
+          raise Fluent::Counter::InvalidParams.new('The type of `name` should be String')
+        end
+
+        unless VALID_NAME =~ name
+          raise Fluent::Counter::InvalidParams.new("`name` is the invalid format")
+        end
+      end
+
+      def validate_value!(hash)
+        value = hash['value']
+        unless value
+          raise Fluent::Counter::InvalidParams.new('`value` is required')
+        end
+
+        unless value.is_a?(Numeric)
+          raise Fluent::Counter::InvalidParams.new("The type of `value` type should be Numeric")
+        end
+      end
+
+      def validate_reset_interval!(hash)
+        interval = hash['reset_interval']
+
+        unless interval
+          raise Fluent::Counter::InvalidParams.new('`reset_interval` is required')
+        end
+
+        unless interval.is_a?(Numeric)
+          raise Fluent::Counter::InvalidParams.new('The type of `reset_interval` should be Numeric')
+        end
+
+        if interval < 0
+          raise Fluent::Counter::InvalidParams.new('`reset_interval` should be a positive number')
+        end
+      end
+    end
+  end
+end

data/lib/fluent/env.rb

@@ -20,6 +20,7 @@ module Fluent
   DEFAULT_CONFIG_PATH = ENV['FLUENT_CONF'] || '/etc/fluent/fluent.conf'
   DEFAULT_PLUGIN_DIR = ENV['FLUENT_PLUGIN'] || '/etc/fluent/plugin'
   DEFAULT_SOCKET_PATH = ENV['FLUENT_SOCKET'] || '/var/run/fluent/fluent.sock'
+  DEFAULT_BACKUP_DIR = ENV['FLUENT_BACKUP_DIR'] || '/tmp/fluent'
   DEFAULT_OJ_OPTIONS = {bigdecimal_load: :float, mode: :compat, use_to_json: true}
 
   def self.windows?

data/lib/fluent/log.rb

@@ -625,5 +625,12 @@ module Fluent
         super
       end
     end
+
+    def reopen(path, mode)
+      if mode != 'a'
+        raise "Unsupported mode: #{mode}"
+      end
+      super(path)
+    end
   end
 end

data/lib/fluent/plugin/filter_grep.rb

@@ -25,10 +25,10 @@ module Fluent::Plugin
     def initialize
       super
 
-      @_regexp_and_conditions = []
-      @_exclude_and_conditions = []
-      @_regexp_or_conditions = []
-      @_exclude_or_conditions = []
+      @_regexp_and_conditions = nil
+      @_exclude_and_conditions = nil
+      @_regexp_or_conditions = nil
+      @_exclude_or_conditions = nil
     end
 
     # for test
@@ -153,35 +153,31 @@ module Fluent::Plugin
         end
       end
 
-      @_regexp_and_conditions = regexp_and_conditions.values
-      @_exclude_and_conditions = exclude_and_conditions.values
-      @_regexp_or_conditions = regexp_or_conditions.values
-      @_exclude_or_conditions = exclude_or_conditions.values
+      @_regexp_and_conditions = regexp_and_conditions.values unless regexp_and_conditions.empty?
+      @_exclude_and_conditions = exclude_and_conditions.values unless exclude_and_conditions.empty?
+      @_regexp_or_conditions = regexp_or_conditions.values unless regexp_or_conditions.empty?
+      @_exclude_or_conditions = exclude_or_conditions.values unless exclude_or_conditions.empty?
     end
 
     def filter(tag, time, record)
-      result = nil
       begin
-        catch(:break_loop) do
-          @_regexp_and_conditions.each do |expression|
-            throw :break_loop unless expression.match?(record)
-          end
-          if !@_regexp_or_conditions.empty? && @_regexp_or_conditions.none? {|expression| expression.match?(record) }
-            throw :break_loop
-          end
-          if !@_exclude_and_conditions.empty? && @_exclude_and_conditions.all? {|expression| expression.match?(record) }
-            throw :break_loop
-          end
-          @_exclude_or_conditions.each do |expression|
-            throw :break_loop if expression.match?(record)
-          end
-          result = record
+        if @_regexp_and_conditions && @_regexp_and_conditions.any? { |expression| !expression.match?(record) }
+          return nil
+        end
+        if @_regexp_or_conditions && @_regexp_or_conditions.none? { |expression| expression.match?(record) }
+          return nil
+        end
+        if @_exclude_and_conditions && @_exclude_and_conditions.all? { |expression| expression.match?(record) }
+          return nil
+        end
+        if @_exclude_or_conditions && @_exclude_or_conditions.any? { |expression| expression.match?(record) }
+          return nil
         end
       rescue => e
         log.warn "failed to grep events", error: e
         log.warn_backtrace
       end
-      result
+      record
     end
 
     Expression = Struct.new(:key, :pattern) do

data/lib/fluent/plugin/output.rb

@@ -14,6 +14,7 @@
 #    limitations under the License.
 #
 
+require 'fluent/error'
 require 'fluent/plugin/base'
 require 'fluent/plugin_helper/record_accessor'
 require 'fluent/log'
@@ -873,7 +874,7 @@ module Fluent
             begin
               oldest = @buffer.dequeue_chunk
               if oldest
-                log.warn "dropping oldest chunk to make space after buffer overflow", chunk_id: oldest.unique_id
+                log.warn "dropping oldest chunk to make space after buffer overflow", chunk_id: dump_unique_id_hex(oldest.unique_id)
                 @buffer.purge_chunk(oldest.unique_id)
               else
                 log.error "no queued chunks to be dropped for drop_oldest_chunk"
@@ -1056,6 +1057,8 @@ module Fluent
         end
       end
 
+      UNRECOVERABLE_ERRORS = [Fluent::UnrecoverableError, TypeError, ArgumentError, NoMethodError]
+
       def try_flush
         chunk = @buffer.dequeue_chunk
         return unless chunk
@@ -1100,6 +1103,37 @@ module Fluent
             commit_write(chunk_id, delayed: false, secondary: using_secondary)
             log.trace "done to commit a chunk", chunk: dump_chunk_id
           end
+        rescue *UNRECOVERABLE_ERRORS => e
+          if @secondary
+            if using_secondary
+              log.warn "got unrecoverable error in secondary.", error: e
+              backup_chunk(chunk, using_secondary, output.delayed_commit)
+            else
+              if (self.class == @secondary.class)
+                log.warn "got unrecoverable error in primary and secondary type is same as primary. Skip secondary", error: e
+                backup_chunk(chunk, using_secondary, output.delayed_commit)
+              else
+                # Call secondary output directly without retry update.
+                # In this case, delayed commit causes inconsistent state in dequeued chunks so async output in secondary is not allowed for now.
+                if @secondary.delayed_commit
+                  log.warn "got unrecoverable error in primary and secondary is async output. Skip secondary for backup", error: e
+                  backup_chunk(chunk, using_secondary, output.delayed_commit)
+                else
+                  log.warn "got unrecoverable error in primary. Skip retry and flush chunk to secondary", error: e
+                  begin
+                    @secondary.write(chunk)
+                    commit_write(chunk_id, delayed: output.delayed_commit, secondary: true)
+                  rescue => e
+                    log.warn "got an error in secondary for unrecoverable error", error: e
+                    backup_chunk(chunk, using_secondary, output.delayed_commit)
+                  end
+                end
+              end
+            end
+          else
+            log.warn "got unrecoverable error in primary and no secondary", error: e
+            backup_chunk(chunk, using_secondary, output.delayed_commit)
+          end
         rescue => e
           log.debug "taking back chunk for errors.", chunk: dump_unique_id_hex(chunk.unique_id)
           if output.delayed_commit
@@ -1115,6 +1149,21 @@ module Fluent
         end
       end
 
+      def backup_chunk(chunk, using_secondary, delayed_commit)
+        unique_id = dump_unique_id_hex(chunk.unique_id)
+        safe_plugin_id = plugin_id.gsub(/[ "\/\\:;|*<>?]/, '_')
+        backup_base_dir = system_config.root_dir || DEFAULT_BACKUP_DIR
+        backup_file = File.join(backup_base_dir, 'backup', "worker#{fluentd_worker_id}", safe_plugin_id, "#{unique_id}.log")
+        backup_dir = File.dirname(backup_file)
+
+        log.warn "bad chunk is moved to #{backup_file}"
+        FileUtils.mkdir_p(backup_dir) unless Dir.exist?(backup_dir)
+        File.open(backup_file, 'ab', system_config.file_permission || 0644) { |f|
+          chunk.write_to(f)
+        }
+        commit_write(chunk.unique_id, secondary: using_secondary, delayed: delayed_commit)
+      end
+
       def check_slow_flush(start)
         elapsed_time = Fluent::Clock.now - start
         if elapsed_time > @slow_flush_log_threshold