fluentd 1.11.3 → 1.12.0

data/lib/fluent/command/ctl.rb

@@ -0,0 +1,177 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'optparse'
+require 'fluent/env'
+require 'fluent/version'
+if Fluent.windows?
+  require 'win32/event'
+  require 'win32/service'
+end
+
+module Fluent
+  class Ctl
+    DEFAULT_OPTIONS = {}
+
+    if Fluent.windows?
+      include Windows::ServiceConstants
+      include Windows::ServiceStructs
+      include Windows::ServiceFunctions
+
+      COMMAND_MAP = {
+        shutdown: "",
+        restart: "HUP",
+        flush: "USR1",
+        reload: "USR2",
+      }
+      WINSVC_CONTROL_CODE_MAP = {
+        shutdown: SERVICE_CONTROL_STOP,
+        # 128 - 255: user-defined control code
+        # See https://docs.microsoft.com/en-us/windows/win32/api/winsvc/nf-winsvc-controlservice
+        restart: 128,
+        flush: 129,
+        reload: SERVICE_CONTROL_PARAMCHANGE,
+      }
+    else
+      COMMAND_MAP = {
+        shutdown: :TERM,
+        restart: :HUP,
+        flush: :USR1,
+        reload: :USR2,
+      }
+    end
+
+    def initialize(argv = ARGV)
+      @argv = argv
+      @options = {}
+      @opt_parser = OptionParser.new
+      configure_option_parser
+      @options.merge!(DEFAULT_OPTIONS)
+      parse_options!
+    end
+
+    def help_text
+      text = "\n"
+      if Fluent.windows?
+        text << "Usage: #{$PROGRAM_NAME} COMMAND [PID_OR_SVCNAME]\n"
+      else
+        text << "Usage: #{$PROGRAM_NAME} COMMAND PID\n"
+      end
+      text << "\n"
+      text << "Commands: \n"
+      COMMAND_MAP.each do |key, value|
+        text << "  #{key}\n"
+      end
+      text
+    end
+
+    def usage(msg = nil)
+      puts help_text
+      if msg
+        puts
+        puts "Error: #{msg}"
+      end
+      exit 1
+    end
+
+    def call
+      if Fluent.windows?
+        if @pid_or_svcname =~ /^[0-9]+$/
+          # Use as PID
+          return call_windows_event(@command, "fluentd_#{@pid_or_svcname}")
+        end
+
+        unless call_winsvc_control_code(@command, @pid_or_svcname)
+          puts "Cannot send control code to #{@pid_or_svcname} service, try to send an event with this name ..."
+          call_windows_event(@command, @pid_or_svcname)
+        end
+      else
+        call_signal(@command, @pid_or_svcname)
+      end
+    end
+
+    private
+
+    def call_signal(command, pid)
+      signal = COMMAND_MAP[command.to_sym]
+      Process.kill(signal, pid.to_i)
+    end
+
+    def call_winsvc_control_code(command, pid_or_svcname)
+      status = SERVICE_STATUS.new
+
+      begin
+        handle_scm = OpenSCManager(nil, nil, SC_MANAGER_CONNECT)
+        FFI.raise_windows_error('OpenSCManager') if handle_scm == 0
+
+        handle_scs = OpenService(handle_scm, "fluentdwinsvc", SERVICE_STOP | SERVICE_PAUSE_CONTINUE | SERVICE_USER_DEFINED_CONTROL)
+        FFI.raise_windows_error('OpenService') if handle_scs == 0
+
+        control_code = WINSVC_CONTROL_CODE_MAP[command.to_sym]
+
+        unless ControlService(handle_scs, control_code, status)
+          FFI.raise_windows_error('ControlService')
+        end
+      rescue => e
+        puts e
+        state = status[:dwCurrentState]
+        return state == SERVICE_STOPPED || state == SERVICE_STOP_PENDING
+      ensure
+        CloseServiceHandle(handle_scs)
+        CloseServiceHandle(handle_scm)
+      end
+
+      return true
+    end
+
+    def call_windows_event(command, pid_or_svcname)
+      prefix = pid_or_svcname
+      event_name = COMMAND_MAP[command.to_sym]
+      suffix = event_name.empty? ? "" : "_#{event_name}"
+
+      begin
+        event = Win32::Event.open("#{prefix}#{suffix}")
+        event.set
+        event.close
+      rescue Errno::ENOENT => e
+        puts "Error: Cannot find the fluentd process with the event name: \"#{prefix}\""
+      end
+    end
+
+    def configure_option_parser
+      @opt_parser.banner = help_text
+      @opt_parser.version = Fluent::VERSION
+    end
+
+    def parse_options!
+      @opt_parser.parse!(@argv)
+
+      @command = @argv[0]
+      @pid_or_svcname = @argv[1] || "fluentdwinsvc"
+
+      usage("Command isn't specified!") if @command.nil? || @command.empty?
+      usage("Unknown command: #{@command}") unless COMMAND_MAP.has_key?(@command.to_sym)
+
+      if Fluent.windows?
+        usage("PID or SVCNAME isn't specified!") if @pid_or_svcname.nil? || @pid_or_svcname.empty?
+      else
+        usage("PID isn't specified!") if @pid_or_svcname.nil? || @pid_or_svcname.empty?
+        usage("Invalid PID: #{pid}") unless @pid_or_svcname =~ /^[0-9]+$/
+      end
+    end
+  end
+end
+

data/lib/fluent/command/plugin_config_formatter.rb

@@ -29,7 +29,8 @@ class FluentPluginConfigFormatter
   AVAILABLE_FORMATS = [:markdown, :txt, :json]
   SUPPORTED_TYPES = [
     "input", "output", "filter",
-    "buffer", "parser", "formatter", "storage"
+    "buffer", "parser", "formatter", "storage",
+    "service_discovery"
   ]
 
   DOCS_BASE_URL = "https://docs.fluentd.org/v/1.0"

data/lib/fluent/env.rb

@@ -28,4 +28,8 @@ module Fluent
   def self.windows?
     ServerEngine.windows?
   end
+
+  def self.linux?
+    /linux/ === RUBY_PLATFORM
+  end
 end

data/lib/fluent/plugin.rb

@@ -121,6 +121,11 @@ module Fluent
       new_impl('sd', SD_REGISTRY, type, parent)
     end
 
+    class << self
+      # This should be defined for fluent-plugin-config-formatter type arguments.
+      alias_method :new_service_discovery, :new_sd
+    end
+
     def self.new_parser(type, parent: nil)
       if type[0] == '/' && type[-1] == '/'
         # This usage is not recommended for new API... create RegexpParser directly

data/lib/fluent/plugin/buffer.rb

@@ -143,33 +143,14 @@ module Fluent
           end
         end
 
-        # timekey should be unixtime as usual.
-        # So, unixtime should be bigger than 2^30 - 1 (= 1073741823) nowadays.
-        # We should check object_id stability to use object_id as optimization for comparing operations.
-        # e.g.)
-        # irb> Time.parse("2020/07/31 18:30:00+09:00").to_i
-        # => 1596187800
-        # irb> Time.parse("2020/07/31 18:30:00+09:00").to_i > 2**30 -1
-        # => true
-        def self.enable_optimize?
-          a1 = 2**30 - 1
-          a2 = 2**30 - 1
-          b1 = 2**62 - 1
-          b2 = 2**62 - 1
-          (a1.object_id == a2.object_id) && (b1.object_id == b2.object_id)
-        end
-
         # This is an optimization code. Current Struct's implementation is comparing all data.
         # https://github.com/ruby/ruby/blob/0623e2b7cc621b1733a760b72af246b06c30cf96/struct.c#L1200-L1203
         # Actually this overhead is very small but this class is generated *per chunk* (and used in hash object).
         # This means that this class is one of the most called object in Fluentd.
         # See https://github.com/fluent/fluentd/pull/2560
-        # But, this optimization has a side effect on Windows and 32bit environment(s) due to differing object_id.
-        # This difference causes flood of buffer files.
-        # So, this optimization should be enabled on `enable_optimize?` as true platforms.
         def hash
-          timekey.object_id
-        end if enable_optimize?
+          timekey.hash
+        end
       end
 
       # for tests

data/lib/fluent/plugin/formatter.rb

@@ -46,5 +46,29 @@ module Fluent
         @proc.call(tag, time, record)
       end
     end
+
+    module Newline
+      module Mixin
+        include Fluent::Configurable
+
+        DEFAULT_NEWLINE = if Fluent.windows?
+                            :crlf
+                          else
+                            :lf
+                          end
+
+        config_param :newline, :enum, list: [:lf, :crlf], default: DEFAULT_NEWLINE
+
+        def configure(conf)
+          super
+          @newline = case newline
+                     when :lf
+                       "\n".freeze
+                     when :crlf
+                       "\r\n".freeze
+                     end
+        end
+      end
+    end
   end
 end

data/lib/fluent/plugin/formatter_csv.rb

@@ -27,7 +27,7 @@ module Fluent
       helpers :record_accessor
 
       config_param :delimiter, default: ',' do |val|
-        ['\t', 'TAB'].include?(val) ? "\t" : val
+        ['\t', 'TAB'].include?(val) ? "\t".freeze : val.freeze
       end
       config_param :force_quotes, :bool, default: true
       # "array" looks good for type of :fields, but this implementation removes tailing comma

data/lib/fluent/plugin/formatter_hash.rb

@@ -19,13 +19,15 @@ require 'fluent/plugin/formatter'
 module Fluent
   module Plugin
     class HashFormatter < Formatter
+      include Fluent::Plugin::Newline::Mixin
+
       Plugin.register_formatter('hash', self)
 
       config_param :add_newline, :bool, default: true
 
       def format(tag, time, record)
         line = record.to_s
-        line << "\n".freeze if @add_newline
+        line << @newline if @add_newline
         line
       end
     end

data/lib/fluent/plugin/formatter_json.rb

@@ -20,6 +20,8 @@ require 'fluent/env'
 module Fluent
   module Plugin
     class JSONFormatter < Formatter
+      include Fluent::Plugin::Newline::Mixin
+
       Plugin.register_formatter('json', self)
 
       config_param :json_parser, :string, default: 'oj'
@@ -44,7 +46,7 @@ module Fluent
       end
 
       def format(tag, time, record)
-        "#{@dump_proc.call(record)}\n"
+        "#{@dump_proc.call(record)}#{@newline}"
       end
 
       def format_without_nl(tag, time, record)

data/lib/fluent/plugin/formatter_ltsv.rb

@@ -19,12 +19,14 @@ require 'fluent/plugin/formatter'
 module Fluent
   module Plugin
     class LabeledTSVFormatter < Formatter
+      include Fluent::Plugin::Newline::Mixin
+
       Plugin.register_formatter('ltsv', self)
 
       # http://ltsv.org/
 
-      config_param :delimiter, :string, default: "\t"
-      config_param :label_delimiter, :string, default: ":"
+      config_param :delimiter, :string, default: "\t".freeze
+      config_param :label_delimiter, :string, default: ":".freeze
       config_param :add_newline, :bool, default: true
 
       # TODO: escaping for \t in values
@@ -34,7 +36,7 @@ module Fluent
           formatted << @delimiter if formatted.length.nonzero?
           formatted << "#{label}#{@label_delimiter}#{value}"
         end
-        formatted << "\n".freeze if @add_newline
+        formatted << @newline if @add_newline
         formatted
       end
     end

data/lib/fluent/plugin/formatter_out_file.rb

@@ -21,15 +21,17 @@ require 'yajl'
 module Fluent
   module Plugin
     class OutFileFormatter < Formatter
+      include Fluent::Plugin::Newline::Mixin
+
       Plugin.register_formatter('out_file', self)
 
       config_param :output_time, :bool, default: true
       config_param :output_tag, :bool, default: true
       config_param :delimiter, default: "\t" do |val|
         case val
-        when /SPACE/i then ' '
-        when /COMMA/i then ','
-        else "\t"
+        when /SPACE/i then ' '.freeze
+        when /COMMA/i then ','.freeze
+        else "\t".freeze
         end
       end
       config_set_default :time_type, :string
@@ -44,7 +46,7 @@ module Fluent
         header = ''
         header << "#{@timef.format(time)}#{@delimiter}" if @output_time
         header << "#{tag}#{@delimiter}" if @output_tag
-        "#{header}#{Yajl.dump(record)}\n"
+        "#{header}#{Yajl.dump(record)}#{@newline}"
       end
     end
   end

data/lib/fluent/plugin/formatter_single_value.rb

@@ -19,14 +19,16 @@ require 'fluent/plugin/formatter'
 module Fluent
   module Plugin
     class SingleValueFormatter < Formatter
+      include Fluent::Plugin::Newline::Mixin
+
       Plugin.register_formatter('single_value', self)
 
-      config_param :message_key, :string, default: 'message'
+      config_param :message_key, :string, default: 'message'.freeze
       config_param :add_newline, :bool, default: true
 
       def format(tag, time, record)
         text = record[@message_key].to_s.dup
-        text << "\n" if @add_newline
+        text << @newline if @add_newline
         text
       end
     end

data/lib/fluent/plugin/formatter_tsv.rb

@@ -19,18 +19,20 @@ require 'fluent/plugin/formatter'
 module Fluent
   module Plugin
     class TSVFormatter < Formatter
+      include Fluent::Plugin::Newline::Mixin
+
       Plugin.register_formatter('tsv', self)
 
       desc 'Field names included in each lines'
       config_param :keys, :array, value_type: :string
       desc 'The delimiter character (or string) of TSV values'
-      config_param :delimiter, :string, default: "\t"
+      config_param :delimiter, :string, default: "\t".freeze
       desc 'The parameter to enable writing to new lines'
       config_param :add_newline, :bool, default: true
 
       def format(tag, time, record)
         formatted = @keys.map{|k| record[k].to_s }.join(@delimiter)
-        formatted << "\n".freeze if @add_newline
+        formatted << @newline if @add_newline
         formatted
       end
     end

data/lib/fluent/plugin/in_http.rb

@@ -80,6 +80,8 @@ module Fluent::Plugin
     config_param :use_204_response, :bool, default: false
     desc 'Dump error log or not'
     config_param :dump_error_log, :bool, default: true
+    desc 'Add QUERY_ prefix query params to record'
+    config_param :add_query_params, :bool, default: false
 
     config_section :parse do
       config_set_default :@type, 'in_http'
@@ -277,7 +279,7 @@ module Fluent::Plugin
     private
 
     def on_server_connect(conn)
-      handler = Handler.new(conn, @km, method(:on_request), @body_size_limit, @format_name, log, @cors_allow_origins)
+      handler = Handler.new(conn, @km, method(:on_request), @body_size_limit, @format_name, log, @cors_allow_origins, @add_query_params)
 
       conn.on(:data) do |data|
         handler.on_read(data)
@@ -326,6 +328,14 @@ module Fluent::Plugin
         }
       end
 
+      if @add_query_params
+        params.each_pair { |k, v|
+          if k.start_with?("QUERY_".freeze)
+            record[k] = v
+          end
+        }
+      end
+
       if @add_remote_addr
         record['REMOTE_ADDR'] = params['REMOTE_ADDR']
       end
@@ -346,7 +356,7 @@ module Fluent::Plugin
     class Handler
       attr_reader :content_type
 
-      def initialize(io, km, callback, body_size_limit, format_name, log, cors_allow_origins)
+      def initialize(io, km, callback, body_size_limit, format_name, log, cors_allow_origins, add_query_params)
         @io = io
         @km = km
         @callback = callback
@@ -356,6 +366,7 @@ module Fluent::Plugin
         @log = log
         @cors_allow_origins = cors_allow_origins
         @idle = 0
+        @add_query_params = add_query_params
         @km.add(self)
 
         @remote_port, @remote_addr = io.remote_port, io.remote_addr
@@ -533,7 +544,17 @@ module Fluent::Plugin
         end
         path_info = uri.path
 
+        if (@add_query_params) 
+
+          query_params = WEBrick::HTTPUtils.parse_query(uri.query)
+
+          query_params.each_pair {|k,v|
+            params["QUERY_#{k.gsub('-','_').upcase}"] = v
+          }
+        end
+
         params.merge!(@env)
+
         @env.clear
 
         code, header, body = @callback.call(path_info, params)