fluentd 0.12.43 → 0.14.0

data/lib/fluent/plugin/storage.rb

@@ -0,0 +1,84 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/plugin/base'
+require 'fluent/plugin/owned_by_mixin'
+
+module Fluent
+  module Plugin
+    class Storage < Base
+      include OwnedByMixin
+
+      DEFAULT_TYPE = 'local'
+
+      configured_in :storage
+
+      config_param :persistent,        :bool, default: false # load/save with all operations
+      config_param :autosave,          :bool, default: true
+      config_param :autosave_interval, :time, default: 10
+      config_param :save_at_shutdown,  :bool, default: true
+
+      def self.validate_key(key)
+        raise ArgumentError, "key must be a string (or symbol for to_s)" unless key.is_a?(String) || key.is_a?(Symbol)
+        key.to_s
+      end
+
+      attr_accessor :log
+
+      def persistent_always?
+        false
+      end
+
+      def synchronized?
+        false
+      end
+
+      def implementation
+        self
+      end
+
+      def load
+        # load storage data from any data source, or initialize storage internally
+      end
+
+      def save
+        # save internal data store into data source (to be loaded)
+      end
+
+      def get(key)
+        raise NotImplementedError, "Implement this method in child class"
+      end
+
+      def fetch(key, defval)
+        raise NotImplementedError, "Implement this method in child class"
+      end
+
+      def put(key, value)
+        # return value
+        raise NotImplementedError, "Implement this method in child class"
+      end
+
+      def delete(key)
+        # return deleted value
+        raise NotImplementedError, "Implement this method in child class"
+      end
+
+      def update(key, &block) # transactional get-and-update
+        raise NotImplementedError, "Implement this method in child class"
+      end
+    end
+  end
+end

data/lib/fluent/plugin/storage_local.rb

@@ -0,0 +1,116 @@
+require 'fluent/plugin'
+require 'fluent/plugin/storage'
+
+require 'fileutils'
+require 'yajl'
+
+module Fluent
+  module Plugin
+    class LocalStorage < Storage
+      Fluent::Plugin.register_storage('local', self)
+
+      DEFAULT_DIR_MODE = 0755
+      DEFAULT_FILE_MODE = 0644
+
+      config_param :path, :string, default: nil
+      config_param :mode, :integer, default: DEFAULT_FILE_MODE
+      config_param :dir_mode, :integer, default: DEFAULT_DIR_MODE
+      config_param :pretty_print, :bool, default: false
+
+      def initialize
+        super
+        @store = {}
+      end
+
+      def configure(conf)
+        super
+
+        @on_memory = false
+        if !@path && !@_plugin_id_configured
+          if @persistent
+            raise Fluent::ConfigError, "Plugin @id or path for <storage> required to save data"
+          else
+            if @autosave
+              log.warn "both of Plugin @id and path for <storage> are not specified. Using on-memory store."
+            else
+              log.info "both of Plugin @id and path for <storage> are not specified. Using on-memory store."
+            end
+            @on_memory = true
+          end
+        elsif @path
+          # ok
+        else # @_plugin_id_configured is true
+          raise NotImplementedError, "implement this feature later with system_config"
+          ## TODO: get process-wide directory for plugin storage, and generate path for this plugin storage instance
+          # path = 
+        end
+
+        if !@on_memory
+          dir = File.dirname(@path)
+          FileUtils.mkdir_p(dir, mode: @dir_mode) unless File.exist?(dir)
+          if File.exist?(@path)
+            raise Fluent::ConfigError, "Plugin storage path '#{@path}' is not readable/writable" unless File.readable?(@path) && File.writable?(@path)
+            begin
+              data = Yajl::Parser.parse(open(@path, 'r:utf-8'){ |io| io.read })
+              raise Fluent::ConfigError, "Invalid contents (not object) in plugin storage file: '#{@path}'" unless data.is_a?(Hash)
+            rescue => e
+              log.error "failed to read data from plugin storage file", path: @path, error: e
+              raise Fluent::ConfigError, "Unexpected error: failed to read data from plugin storage file: '#{@path}'"
+            end
+          else
+            raise Fluent::ConfigError, "Directory is not writable for plugin storage file '#{@path}'" unless File.writable?(@path)
+          end
+        end
+      end
+
+      def load
+        return if @on_memory
+        return unless File.exist?(@path)
+        begin
+          json_string = open(@path, 'r:utf-8'){ |io| io.read }
+          json = Yajl::Parser.parse(json_string)
+          unless json.is_a?(Hash)
+            log.error "broken content for plugin storage (Hash required: ignored)", type: json.class
+            log.debug "broken content", content: json_string
+            return
+          end
+          @store = json
+        rescue => e
+          log.error "failed to load data for plugin storage from file", path: @path, error: e
+        end
+      end
+
+      def save
+        return if @on_memory
+        tmp_path = @path + '.tmp'
+        begin
+          json_string = Yajl::Encoder.encode(@store, pretty: @pretty_print)
+          open(tmp_path, 'w:utf-8', @mode){ |io| io.write json_string }
+          File.rename(tmp_path, @path)
+        rescue => e
+          log.error "failed to save data for plugin storage to file", path: @path, tmp: tmp_path, error: e
+        end
+      end
+
+      def get(key)
+        @store[key.to_s]
+      end
+
+      def fetch(key, defval)
+        @store.fetch(key.to_s, defval)
+      end
+
+      def put(key, value)
+        @store[key.to_s] = value
+      end
+
+      def delete(key)
+        @store.delete(key.to_s)
+      end
+
+      def update(key, &block)
+        @store[key.to_s] = block.call(@store[key.to_s])
+      end
+    end
+  end
+end

data/lib/fluent/plugin/string_util.rb

@@ -14,21 +14,24 @@
 #    limitations under the License.
 #
 
-require "string-scrub" unless "".respond_to?(:scrub)
-
 module Fluent
-  module StringUtil
-    def match_regexp(regexp, string)
-      begin
-        return regexp.match(string)
-      rescue ArgumentError => e
-        raise e unless e.message.index("invalid byte sequence in".freeze).zero?
-        $log.info "invalid byte sequence is replaced in `#{string}`"
-        string = string.scrub('?')
-        retry
+  module Plugin
+    module StringUtil
+      def match_regexp(regexp, string)
+        begin
+          return regexp.match(string)
+        rescue ArgumentError => e
+          raise e unless e.message.index("invalid byte sequence in".freeze).zero?
+          $log.info "invalid byte sequence is replaced in `#{string}`"
+          string = string.scrub('?')
+          retry
+        end
+        return true
       end
-      return true
+      module_function :match_regexp
     end
-    module_function :match_regexp
   end
+
+  # obsolete
+  StringUtil = Fluent::Plugin::StringUtil
 end

data/lib/fluent/plugin_helper.rb

@@ -0,0 +1,39 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/plugin_helper/event_emitter'
+require 'fluent/plugin_helper/thread'
+require 'fluent/plugin_helper/event_loop'
+require 'fluent/plugin_helper/timer'
+require 'fluent/plugin_helper/child_process'
+require 'fluent/plugin_helper/storage'
+require 'fluent/plugin_helper/retry_state'
+require 'fluent/plugin_helper/compat_parameters'
+
+module Fluent
+  module PluginHelper
+    module Mixin
+      def self.included(mod)
+        mod.extend(Fluent::PluginHelper)
+      end
+    end
+
+    def helpers(*snake_case_symbols)
+      helper_modules = snake_case_symbols.map{|name| Fluent::PluginHelper.const_get(name.to_s.split('_').map(&:capitalize).join) }
+      include(*helper_modules)
+    end
+  end
+end

data/lib/fluent/plugin_helper/child_process.rb

@@ -0,0 +1,298 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/plugin_helper/thread'
+require 'fluent/plugin_helper/timer'
+
+require 'open3'
+require 'timeout'
+
+module Fluent
+  module PluginHelper
+    module ChildProcess
+      include Fluent::PluginHelper::Thread
+      include Fluent::PluginHelper::Timer
+
+      CHILD_PROCESS_LOOP_CHECK_INTERVAL = 0.2 # sec
+      CHILD_PROCESS_DEFAULT_EXIT_TIMEOUT = 10 # sec
+      CHILD_PROCESS_DEFAULT_KILL_TIMEOUT = 60 # sec
+
+      MODE_PARAMS = [:read, :write, :stderr, :read_with_stderr]
+      STDERR_OPTIONS = [:discard, :connect]
+
+      # stop     : mark callback thread as stopped
+      # shutdown : close write IO to child processes (STDIN of child processes), send TERM (KILL for Windows) to all child processes
+      # close    : send KILL to all child processes
+      # terminate: [-]
+
+      attr_reader :_child_process_processes # for tests
+
+      def child_process_running?
+        # checker for code in callback of child_process_execute
+        ::Thread.current[:_fluentd_plugin_helper_child_process_running] || false
+      end
+
+      def child_process_id
+        ::Thread.current[:_fluentd_plugin_helper_child_process_pid]
+      end
+
+      def child_process_exit_status
+        ::Thread.current[:_fluentd_plugin_helper_child_process_exit_status]
+      end
+
+      def child_process_execute(
+          title, command,
+          arguments: nil, subprocess_name: nil, interval: nil, immediate: false, parallel: false,
+          mode: [:read, :write], stderr: :discard, env: {}, unsetenv: false, chdir: nil,
+          internal_encoding: 'utf-8', external_encoding: 'ascii-8bit', scrub: true, replace_string: nil,
+          &block
+      )
+        raise ArgumentError, "BUG: title must be a symbol" unless title.is_a? Symbol
+        raise ArgumentError, "BUG: arguments required if subprocess name is replaced" if subprocess_name && !arguments
+
+        raise ArgumentError, "BUG: invalid mode specification" unless mode.all?{|m| MODE_PARAMS.include?(m) }
+        raise ArgumentError, "BUG: read_with_stderr is exclusive with :read and :stderr" if mode.include?(:read_with_stderr) && (mode.include?(:read) || mode.include?(:stderr))
+        raise ArgumentError, "BUG: invalid stderr handling specification" unless STDERR_OPTIONS.include?(stderr)
+
+        raise ArgumentError, "BUG: block not specified which receive i/o object" unless block_given?
+        raise ArgumentError, "BUG: number of block arguments are different from size of mode" unless block.arity == mode.size
+
+        running = false
+        callback = ->(*args) {
+          running = true
+          begin
+            block.call(*args)
+          ensure
+            running = false
+          end
+        }
+
+        if immediate || !interval
+          child_process_execute_once(title, command, arguments, subprocess_name, mode, stderr, env, unsetenv, chdir, internal_encoding, external_encoding, scrub, replace_string, &callback)
+        end
+
+        if interval
+          timer_execute(:child_process_execute, interval, repeat: true) do
+            if !parallel && running
+              log.warn "previous child process is still running. skipped.", title: title, command: command, arguments: arguments, interval: interval, parallel: parallel
+            else
+              child_process_execute_once(title, command, arguments, subprocess_name, mode, stderr, env, unsetenv, chdir, internal_encoding, external_encoding, scrub, replace_string, &callback)
+            end
+          end
+        end
+      end
+
+      def initialize
+        super
+        # plugins MAY configure this parameter
+        @_child_process_exit_timeout = CHILD_PROCESS_DEFAULT_EXIT_TIMEOUT
+        @_child_process_kill_timeout = CHILD_PROCESS_DEFAULT_KILL_TIMEOUT
+        @_child_process_mutex = Mutex.new
+      end
+
+      def start
+        super
+        @_child_process_processes = {} # pid => ProcessInfo
+      end
+
+      def stop
+        @_child_process_mutex.synchronize{ @_child_process_processes.keys }.each do |pid|
+          process_info = @_child_process_processes[pid]
+          if process_info
+            process_info.thread[:_fluentd_plugin_helper_child_process_running] = false
+          end
+        end
+      end
+
+      def shutdown
+        @_child_process_mutex.synchronize{ @_child_process_processes.keys }.each do |pid|
+          process_info = @_child_process_processes[pid]
+          next if !process_info || !process_info.writeio_in_use
+          begin
+            Timeout.timeout(@_child_process_exit_timeout) do
+              process_info.writeio.close
+            end
+          rescue Timeout::Error
+            log.debug "External process #{process_info.title} doesn't exist after STDIN close in timeout #{@_child_process_exit_timeout}sec"
+          end
+
+          child_process_kill(process_info)
+        end
+
+        super
+      end
+
+      def close
+        while (pids = @_child_process_mutex.synchronize{ @_child_process_processes.keys }).size > 0
+          pids.each do |pid|
+            process_info = @_child_process_processes[pid]
+            if !process_info || !process_info.alive
+              @_child_process_mutex.synchronize{ @_child_process_processes.delete(pid) }
+              next
+            end
+
+            process_info.killed_at ||= Time.now # for illegular case (e.g., created after shutdown)
+            next if Time.now < process_info.killed_at + @_child_process_kill_timeout
+
+            child_process_kill(process_info, force: true)
+            @_child_process_mutex.synchronize{ @_child_process_processes.delete(pid) }
+          end
+
+          sleep CHILD_PROCESS_LOOP_CHECK_INTERVAL
+        end
+
+        super
+      end
+
+      def child_process_kill(process_info, force: false)
+        if !process_info || !process_info.alive
+          return
+        end
+
+        process_info.killed_at = Time.now unless force
+
+        begin
+          pid, status = Process.waitpid2(process_info.pid, Process::WNOHANG)
+          if pid && status
+            process_info.thread[:_fluentd_plugin_helper_child_process_exit_status] = status
+            process_info.alive = false
+          end
+        rescue Errno::ECHILD, Errno::ESRCH, Errno::EPERM
+          process_info.alive = false
+        rescue
+          # ignore
+        end
+        if !process_info.alive
+          return
+        end
+
+        begin
+          signal = (Fluent.windows? || force) ? :KILL : :TERM
+          Process.kill(signal, process_info.pid)
+          if force
+            process_info.alive = false
+          end
+        rescue Errno::ECHILD, Errno::ESRCH
+          process_info.alive = false
+        end
+      end
+
+      ProcessInfo = Struct.new(:title, :thread, :pid, :readio, :readio_in_use, :writeio, :writeio_in_use, :stderrio, :stderrio_in_use, :wait_thread, :alive, :killed_at)
+
+      def child_process_execute_once(
+          title, command, arguments, subprocess_name, mode, stderr, env, unsetenv, chdir,
+          internal_encoding, external_encoding, scrub, replace_string, &block
+      )
+        spawn_args = if arguments || subprocess_name
+                       [ env, (subprocess_name ? [command, subprocess_name] : command), *(arguments || []) ]
+                     else
+                       [ env, command ]
+                     end
+        spawn_opts = {
+          unsetenv_others: unsetenv,
+        }
+        if chdir
+          spawn_opts[:chdir] = chdir
+        end
+
+        encoding_options = {}
+        if scrub
+          encoding_options[:invalid] = encoding_options[:undef] = :replace
+          if replace_string
+            encoding_options[:replace] = replace_string
+          end
+        end
+
+        log.debug "Executing command", title: title, spawn: spawn_args, mode: mode, stderr: stderr
+
+        readio = writeio = stderrio = wait_thread = nil
+        readio_in_use = writeio_in_use = stderrio_in_use = false
+
+        if !mode.include?(:stderr) && !mode.include?(:read_with_stderr) && stderr != :discard # connect
+          writeio, readio, wait_thread = *Open3.popen2(*spawn_args, spawn_opts)
+        elsif mode.include?(:read_with_stderr)
+          writeio, readio, wait_thread = *Open3.popen2e(*spawn_args, spawn_opts)
+        else
+          writeio, readio, stderrio, wait_thread = *Open3.popen3(*spawn_args, spawn_opts)
+          if !mode.include?(:stderr) # stderr == :discard
+            stderrio.reopen(IO::NULL)
+          end
+        end
+
+        if mode.include?(:write)
+          writeio.set_encoding(external_encoding, internal_encoding, encoding_options)
+          writeio_in_use = true
+        end
+        if mode.include?(:read) || mode.include?(:read_with_stderr)
+          readio.set_encoding(external_encoding, internal_encoding, encoding_options)
+          readio_in_use = true
+        end
+        if mode.include?(:stderr)
+          stderrio.set_encoding(external_encoding, internal_encoding, encoding_options)
+          stderrio_in_use = true
+        end
+
+        pid = wait_thread.pid # wait_thread => Process::Waiter
+
+        io_objects = []
+        mode.each do |m|
+          io_objects << case m
+                        when :read then readio
+                        when :write then writeio
+                        when :read_with_stderr then readio
+                        when :stderr then stderrio
+                        else
+                          raise "BUG: invalid mode must be checked before here: '#{m}'"
+                        end
+        end
+
+        m = Mutex.new
+        m.lock
+        thread = thread_create :child_process_callback do
+          m.lock # run after plugin thread get pid, thread instance and i/o
+          m.unlock
+          begin
+            block.call(*io_objects)
+          rescue EOFError => e
+            log.debug "Process exit and I/O closed", title: title, pid: pid, command: command, arguments: arguments
+          rescue IOError => e
+            if e.message == 'stream closed'
+              log.debug "Process I/O stream closed", title: title, pid: pid, command: command, arguments: arguments
+            else
+              log.error "Unexpected I/O error for child process", title: title, pid: pid, command: command, arguments: arguments, error: e
+            end
+          rescue => e
+            log.warn "Unexpected error while processing I/O for child process", title: title, pid: pid, command: command, error: e
+          end
+          process_info = @_child_process_mutex.synchronize do
+            process_info = @_child_process_processes[pid]
+            @_child_process_processes.delete(pid)
+            process_info
+          end
+          child_process_kill(process_info, force: true) if process_info && process_info.alive && ::Thread.current[:_fluentd_plugin_helper_child_process_running]
+        end
+        thread[:_fluentd_plugin_helper_child_process_running] = true
+        thread[:_fluentd_plugin_helper_child_process_pid] = pid
+        pinfo = ProcessInfo.new(title, thread, pid, readio, readio_in_use, writeio, writeio_in_use, stderrio, stderrio_in_use, wait_thread, true, nil)
+        @_child_process_mutex.synchronize do
+          @_child_process_processes[pid] = pinfo
+        end
+        m.unlock
+        pid
+      end
+    end
+  end
+end