fluentd222 1.16.2-x86_64-linux

data/lib/fluent/plugin/sd_srv.rb

@@ -0,0 +1,135 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'resolv'
+
+require 'fluent/plugin_helper'
+require 'fluent/plugin/service_discovery'
+
+module Fluent
+  module Plugin
+    class SrvServiceDiscovery < ServiceDiscovery
+      include PluginHelper::Mixin
+
+      Plugin.register_sd('srv', self)
+
+      helpers :timer
+
+      desc 'Service without underscore in RFC2782'
+      config_param :service, :string
+      desc 'Proto without underscore in RFC2782'
+      config_param :proto, :string, default: 'tcp'
+      desc 'Name in RFC2782'
+      config_param :hostname, :string
+      desc 'hostname of DNS server to request the SRV record'
+      config_param :dns_server_host, :string, default: nil
+      desc 'interval of requesting to DNS server'
+      config_param :interval, :integer, default: 60
+      desc "resolve hostname to IP addr of SRV's Target"
+      config_param :dns_lookup, :bool, default: true
+      desc 'The shared key per server'
+      config_param :shared_key, :string, default: nil, secret: true
+      desc 'The username for authentication'
+      config_param :username, :string, default: ''
+      desc 'The password for authentication'
+      config_param :password, :string, default: '', secret: true
+
+      def initialize
+        super
+        @target = nil
+      end
+
+      def configure(conf)
+        super
+
+        @target = "_#{@service}._#{@proto}.#{@hostname}"
+        @dns_resolve =
+          if @dns_server_host.nil?
+            Resolv::DNS.new
+          elsif @dns_server_host.include?(':') # e.g. 127.0.0.1:8600
+            host, port = @dns_server_host.split(':', 2)
+            Resolv::DNS.new(nameserver_port: [[host, port.to_i]])
+          else
+            Resolv::DNS.new(nameserver: @dns_server_host)
+          end
+
+        @services = fetch_srv_record
+      end
+
+      def start(queue)
+        timer_execute(:"sd_srv_record_#{@target}", @interval) do
+          refresh_srv_records(queue)
+        end
+
+        super()
+      end
+
+      private
+
+      def refresh_srv_records(queue)
+        s = begin
+              fetch_srv_record
+            rescue => e
+              @log.error("sd_srv: #{e}")
+              return
+            end
+
+        if s.nil? || s.empty?
+          return
+        end
+
+        diff = []
+        join = s - @services
+        # Need service_in first to guarantee that server exist at least one all time.
+        join.each do |j|
+          diff << ServiceDiscovery.service_in_msg(j)
+        end
+
+        drain = @services - s
+        drain.each do |d|
+          diff << ServiceDiscovery.service_out_msg(d)
+        end
+
+        @services = s
+
+        diff.each do |a|
+          queue.push(a)
+        end
+      end
+
+      def fetch_srv_record
+        adders = @dns_resolve.getresources(@target, Resolv::DNS::Resource::IN::SRV)
+
+        services = []
+
+        adders.each do |addr|
+          host = @dns_lookup ? dns_lookup!(addr.target) : addr.target
+          services << [
+            addr.priority,
+            Service.new(:srv, host.to_s, addr.port.to_i, addr.target.to_s, addr.weight, false, @username, @password, @shared_key)
+          ]
+        end
+
+        services.sort_by(&:first).flat_map { |s| s[1] }
+      end
+
+      def dns_lookup!(host)
+        # may need to cache the result
+        @dns_resolve.getaddress(host) # get first result for now
+      end
+    end
+  end
+end

data/lib/fluent/plugin/sd_static.rb

@@ -0,0 +1,58 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/plugin/service_discovery'
+
+module Fluent
+  module Plugin
+    class StaticServiceDiscovery < ServiceDiscovery
+      Plugin.register_sd('static', self)
+
+      LISTEN_PORT = 24224
+
+      config_section :service, param_name: :service_configs do
+        desc 'The IP address or host name of the server.'
+        config_param :host, :string
+        desc 'The name of the server. Used for logging and certificate verification in TLS transport (when host is address).'
+        config_param :name, :string, default: nil
+        desc 'The port number of the host.'
+        config_param :port, :integer, default: LISTEN_PORT
+        desc 'The shared key per server.'
+        config_param :shared_key, :string, default: nil, secret: true
+        desc 'The username for authentication.'
+        config_param :username, :string, default: ''
+        desc 'The password for authentication.'
+        config_param :password, :string, default: '', secret: true
+        desc 'Marks a node as the standby node for an Active-Standby model between Fluentd nodes.'
+        config_param :standby, :bool, default: false
+        desc 'The load balancing weight.'
+        config_param :weight, :integer, default: 60
+      end
+
+      def configure(conf)
+        super
+
+        @services = @service_configs.map do |s|
+          ServiceDiscovery::Service.new(:static, s.host, s.port, s.name, s.weight, s.standby, s.username, s.password, s.shared_key)
+        end
+      end
+
+      def start(queue = nil)
+        super()
+      end
+    end
+  end
+end

data/lib/fluent/plugin/service_discovery.rb

@@ -0,0 +1,65 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/plugin/base'
+
+require 'fluent/log'
+require 'fluent/unique_id'
+require 'fluent/plugin_id'
+
+module Fluent
+  module Plugin
+    class ServiceDiscovery < Base
+      include PluginId
+      include PluginLoggerMixin
+      include UniqueId::Mixin
+
+      configured_in :service_discovery
+
+      attr_reader :services
+
+      Service = Struct.new(:plugin_name, :host, :port, :name, :weight, :standby, :username, :password, :shared_key) do
+        def discovery_id
+          @discovery_id ||= Base64.encode64(to_h.to_s)
+        end
+      end
+
+      SERVICE_IN = :service_in
+      SERVICE_OUT = :service_out
+      DiscoveryMessage = Struct.new(:type, :service)
+
+      class << self
+        def service_in_msg(service)
+          DiscoveryMessage.new(SERVICE_IN, service)
+        end
+
+        def service_out_msg(service)
+          DiscoveryMessage.new(SERVICE_OUT, service)
+        end
+      end
+
+      def initialize
+        @services = []
+
+        super
+      end
+
+      def start(queue = nil)
+        super()
+      end
+    end
+  end
+end

data/lib/fluent/plugin/socket_util.rb

@@ -0,0 +1,22 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/compat/socket_util'
+
+module Fluent
+  # obsolete
+  SocketUtil = Fluent::Compat::SocketUtil
+end

data/lib/fluent/plugin/storage.rb

@@ -0,0 +1,84 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/plugin/base'
+require 'fluent/plugin/owned_by_mixin'
+
+module Fluent
+  module Plugin
+    class Storage < Base
+      include OwnedByMixin
+
+      DEFAULT_TYPE = 'local'
+
+      configured_in :storage
+
+      config_param :persistent,        :bool, default: false # load/save with all operations
+      config_param :autosave,          :bool, default: true
+      config_param :autosave_interval, :time, default: 10
+      config_param :save_at_shutdown,  :bool, default: true
+
+      def self.validate_key(key)
+        raise ArgumentError, "key must be a string (or symbol for to_s)" unless key.is_a?(String) || key.is_a?(Symbol)
+        key.to_s
+      end
+
+      attr_accessor :log
+
+      def persistent_always?
+        false
+      end
+
+      def synchronized?
+        false
+      end
+
+      def implementation
+        self
+      end
+
+      def load
+        # load storage data from any data source, or initialize storage internally
+      end
+
+      def save
+        # save internal data store into data source (to be loaded)
+      end
+
+      def get(key)
+        raise NotImplementedError, "Implement this method in child class"
+      end
+
+      def fetch(key, defval)
+        raise NotImplementedError, "Implement this method in child class"
+      end
+
+      def put(key, value)
+        # return value
+        raise NotImplementedError, "Implement this method in child class"
+      end
+
+      def delete(key)
+        # return deleted value
+        raise NotImplementedError, "Implement this method in child class"
+      end
+
+      def update(key, &block) # transactional get-and-update
+        raise NotImplementedError, "Implement this method in child class"
+      end
+    end
+  end
+end

data/lib/fluent/plugin/storage_local.rb

@@ -0,0 +1,162 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/env'
+require 'fluent/plugin'
+require 'fluent/plugin/storage'
+
+require 'fileutils'
+require 'yajl'
+
+module Fluent
+  module Plugin
+    class LocalStorage < Storage
+      Fluent::Plugin.register_storage('local', self)
+
+      config_param :path, :string, default: nil
+      config_param :mode, default: Fluent::DEFAULT_FILE_PERMISSION do |v|
+        v.to_i(8)
+      end
+      config_param :dir_mode, default: Fluent::DEFAULT_DIR_PERMISSION do |v|
+        v.to_i(8)
+      end
+      config_param :pretty_print, :bool, default: false
+
+      attr_reader :store # for test
+
+      def initialize
+        super
+        @store = {}
+        @multi_workers_available = nil
+      end
+
+      def configure(conf)
+        super
+
+        @on_memory = false
+        if @path
+          if File.exist?(@path) && File.file?(@path)
+            @multi_workers_available = false
+          elsif File.exist?(@path) && File.directory?(@path)
+            @path = File.join(@path, "worker#{fluentd_worker_id}", "storage.json")
+            @multi_workers_available = true
+          else # path file/directory doesn't exist
+            if @path.end_with?('.json') # file
+              @multi_workers_available = false
+            else # directory
+              @path = File.join(@path, "worker#{fluentd_worker_id}", "storage.json")
+              @multi_workers_available = true
+            end
+          end
+        elsif root_dir = owner.plugin_root_dir
+          basename = (conf.arg && !conf.arg.empty?) ? "storage.#{conf.arg}.json" : "storage.json"
+          @path = File.join(root_dir, basename)
+          @multi_workers_available = true
+        else
+          if @persistent
+            raise Fluent::ConfigError, "Plugin @id or path for <storage> required when 'persistent' is true"
+          else
+            if @autosave
+              log.warn "both of Plugin @id and path for <storage> are not specified. Using on-memory store."
+            else
+              log.info "both of Plugin @id and path for <storage> are not specified. Using on-memory store."
+            end
+            @on_memory = true
+            @multi_workers_available = true
+          end
+        end
+
+        if !@on_memory
+          dir = File.dirname(@path)
+          FileUtils.mkdir_p(dir, mode: @dir_mode) unless Dir.exist?(dir)
+          if File.exist?(@path)
+            raise Fluent::ConfigError, "Plugin storage path '#{@path}' is not readable/writable" unless File.readable?(@path) && File.writable?(@path)
+            begin
+              data = File.open(@path, 'r:utf-8') { |io| io.read }
+              if data.empty?
+                log.warn "detect empty plugin storage file during startup. Ignored: #{@path}"
+                return
+              end
+              data = Yajl::Parser.parse(data)
+              raise Fluent::ConfigError, "Invalid contents (not object) in plugin storage file: '#{@path}'" unless data.is_a?(Hash)
+            rescue => e
+              log.error "failed to read data from plugin storage file", path: @path, error: e
+              raise Fluent::ConfigError, "Unexpected error: failed to read data from plugin storage file: '#{@path}'"
+            end
+          else
+            raise Fluent::ConfigError, "Directory is not writable for plugin storage file '#{@path}'" unless File.stat(dir).writable?
+          end
+        end
+      end
+
+      def multi_workers_ready?
+        unless @multi_workers_available
+          log.error "local plugin storage with multi workers should be configured to use directory 'path', or system root_dir and plugin id"
+        end
+        @multi_workers_available
+      end
+
+      def load
+        return if @on_memory
+        return unless File.exist?(@path)
+        begin
+          json_string = File.open(@path, 'r:utf-8'){ |io| io.read }
+          json = Yajl::Parser.parse(json_string)
+          unless json.is_a?(Hash)
+            log.error "broken content for plugin storage (Hash required: ignored)", type: json.class
+            log.debug "broken content", content: json_string
+            return
+          end
+          @store = json
+        rescue => e
+          log.error "failed to load data for plugin storage from file", path: @path, error: e
+        end
+      end
+
+      def save
+        return if @on_memory
+        tmp_path = @path + '.tmp.' + Fluent::UniqueId.hex(Fluent::UniqueId.generate)
+        begin
+          json_string = Yajl::Encoder.encode(@store, pretty: @pretty_print)
+          File.open(tmp_path, 'w:utf-8', @mode) { |io| io.write json_string; io.fsync }
+          File.rename(tmp_path, @path)
+        rescue => e
+          log.error "failed to save data for plugin storage to file", path: @path, tmp: tmp_path, error: e
+        end
+      end
+
+      def get(key)
+        @store[key.to_s]
+      end
+
+      def fetch(key, defval)
+        @store.fetch(key.to_s, defval)
+      end
+
+      def put(key, value)
+        @store[key.to_s] = value
+      end
+
+      def delete(key)
+        @store.delete(key.to_s)
+      end
+
+      def update(key, &block)
+        @store[key.to_s] = block.call(@store[key.to_s])
+      end
+    end
+  end
+end

data/lib/fluent/plugin/string_util.rb

@@ -0,0 +1,22 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/compat/string_util'
+
+module Fluent
+  # obsolete
+  StringUtil = Fluent::Compat::StringUtil
+end