fluentd-hubspot 0.14.14.1

data/lib/fluent/plugin/parser_tsv.rb

@@ -0,0 +1,42 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/plugin/parser'
+
+module Fluent
+  module Plugin
+    class TSVParser < Parser
+      Plugin.register_parser('tsv', self)
+
+      desc 'Names of fields included in each lines'
+      config_param :keys, :array, value_type: :string
+      desc 'The delimiter character (or string) of TSV values'
+      config_param :delimiter, :string, default: "\t"
+
+      def configure(conf)
+        super
+        @key_num = @keys.length
+      end
+
+      def parse(text)
+        values = text.split(@delimiter, @key_num)
+        r = Hash[@keys.zip(values)]
+        time, record = convert_values(parse_time(r), r)
+        yield time, record
+      end
+    end
+  end
+end

data/lib/fluent/plugin/socket_util.rb

@@ -0,0 +1,22 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/compat/socket_util'
+
+module Fluent
+  # obsolete
+  SocketUtil = Fluent::Compat::SocketUtil
+end

data/lib/fluent/plugin/storage.rb

@@ -0,0 +1,84 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/plugin/base'
+require 'fluent/plugin/owned_by_mixin'
+
+module Fluent
+  module Plugin
+    class Storage < Base
+      include OwnedByMixin
+
+      DEFAULT_TYPE = 'local'
+
+      configured_in :storage
+
+      config_param :persistent,        :bool, default: false # load/save with all operations
+      config_param :autosave,          :bool, default: true
+      config_param :autosave_interval, :time, default: 10
+      config_param :save_at_shutdown,  :bool, default: true
+
+      def self.validate_key(key)
+        raise ArgumentError, "key must be a string (or symbol for to_s)" unless key.is_a?(String) || key.is_a?(Symbol)
+        key.to_s
+      end
+
+      attr_accessor :log
+
+      def persistent_always?
+        false
+      end
+
+      def synchronized?
+        false
+      end
+
+      def implementation
+        self
+      end
+
+      def load
+        # load storage data from any data source, or initialize storage internally
+      end
+
+      def save
+        # save internal data store into data source (to be loaded)
+      end
+
+      def get(key)
+        raise NotImplementedError, "Implement this method in child class"
+      end
+
+      def fetch(key, defval)
+        raise NotImplementedError, "Implement this method in child class"
+      end
+
+      def put(key, value)
+        # return value
+        raise NotImplementedError, "Implement this method in child class"
+      end
+
+      def delete(key)
+        # return deleted value
+        raise NotImplementedError, "Implement this method in child class"
+      end
+
+      def update(key, &block) # transactional get-and-update
+        raise NotImplementedError, "Implement this method in child class"
+      end
+    end
+  end
+end

data/lib/fluent/plugin/storage_local.rb

@@ -0,0 +1,159 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/plugin'
+require 'fluent/plugin/storage'
+
+require 'fileutils'
+require 'yajl'
+
+module Fluent
+  module Plugin
+    class LocalStorage < Storage
+      Fluent::Plugin.register_storage('local', self)
+
+      DEFAULT_DIR_MODE = 0755
+      DEFAULT_FILE_MODE = 0644
+
+      config_param :path, :string, default: nil
+      config_param :mode, default: DEFAULT_FILE_MODE do |v|
+        v.to_i(8)
+      end
+      config_param :dir_mode, default: DEFAULT_DIR_MODE do |v|
+        v.to_i(8)
+      end
+      config_param :pretty_print, :bool, default: false
+
+      attr_reader :store # for test
+
+      def initialize
+        super
+        @store = {}
+        @multi_workers_available = nil
+      end
+
+      def configure(conf)
+        super
+
+        @on_memory = false
+        if @path
+          if File.exist?(@path) && File.file?(@path)
+            @multi_workers_available = false
+          elsif File.exist?(@path) && File.directory?(@path)
+            @path = File.join(@path, "worker#{fluentd_worker_id}", "storage.json")
+            @multi_workers_available = true
+          else # path file/directory doesn't exist
+            if @path.end_with?('.json') # file
+              @multi_workers_available = false
+            else # directory
+              @path = File.join(@path, "worker#{fluentd_worker_id}", "storage.json")
+              @multi_workers_available = true
+            end
+          end
+        elsif root_dir = owner.plugin_root_dir
+          basename = (conf.arg && !conf.arg.empty?) ? "storage.#{conf.arg}.json" : "storage.json"
+          @path = File.join(root_dir, basename)
+          @multi_workers_available = true
+        else
+          if @persistent
+            raise Fluent::ConfigError, "Plugin @id or path for <storage> required when 'persistent' is true"
+          else
+            if @autosave
+              log.warn "both of Plugin @id and path for <storage> are not specified. Using on-memory store."
+            else
+              log.info "both of Plugin @id and path for <storage> are not specified. Using on-memory store."
+            end
+            @on_memory = true
+            @multi_workers_available = true
+          end
+        end
+
+        if !@on_memory
+          dir = File.dirname(@path)
+          FileUtils.mkdir_p(dir, mode: @dir_mode) unless Dir.exist?(dir)
+          if File.exist?(@path)
+            raise Fluent::ConfigError, "Plugin storage path '#{@path}' is not readable/writable" unless File.readable?(@path) && File.writable?(@path)
+            begin
+              data = Yajl::Parser.parse(open(@path, 'r:utf-8'){ |io| io.read })
+              raise Fluent::ConfigError, "Invalid contents (not object) in plugin storage file: '#{@path}'" unless data.is_a?(Hash)
+            rescue => e
+              log.error "failed to read data from plugin storage file", path: @path, error: e
+              raise Fluent::ConfigError, "Unexpected error: failed to read data from plugin storage file: '#{@path}'"
+            end
+          else
+            raise Fluent::ConfigError, "Directory is not writable for plugin storage file '#{@path}'" unless File.stat(dir).writable?
+          end
+        end
+      end
+
+      def multi_workers_ready?
+        unless @multi_workers_available
+          log.error "local plugin storage with multi workers should be configured to use directory 'path', or system root_dir and plugin id"
+        end
+        @multi_workers_available
+      end
+
+      def load
+        return if @on_memory
+        return unless File.exist?(@path)
+        begin
+          json_string = open(@path, 'r:utf-8'){ |io| io.read }
+          json = Yajl::Parser.parse(json_string)
+          unless json.is_a?(Hash)
+            log.error "broken content for plugin storage (Hash required: ignored)", type: json.class
+            log.debug "broken content", content: json_string
+            return
+          end
+          @store = json
+        rescue => e
+          log.error "failed to load data for plugin storage from file", path: @path, error: e
+        end
+      end
+
+      def save
+        return if @on_memory
+        tmp_path = @path + '.tmp'
+        begin
+          json_string = Yajl::Encoder.encode(@store, pretty: @pretty_print)
+          open(tmp_path, 'w:utf-8', @mode){ |io| io.write json_string }
+          File.rename(tmp_path, @path)
+        rescue => e
+          log.error "failed to save data for plugin storage to file", path: @path, tmp: tmp_path, error: e
+        end
+      end
+
+      def get(key)
+        @store[key.to_s]
+      end
+
+      def fetch(key, defval)
+        @store.fetch(key.to_s, defval)
+      end
+
+      def put(key, value)
+        @store[key.to_s] = value
+      end
+
+      def delete(key)
+        @store.delete(key.to_s)
+      end
+
+      def update(key, &block)
+        @store[key.to_s] = block.call(@store[key.to_s])
+      end
+    end
+  end
+end

data/lib/fluent/plugin/string_util.rb

@@ -0,0 +1,22 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/compat/string_util'
+
+module Fluent
+  # obsolete
+  StringUtil = Fluent::Compat::StringUtil
+end

data/lib/fluent/plugin_helper.rb

@@ -0,0 +1,70 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/plugin_helper/event_emitter'
+require 'fluent/plugin_helper/thread'
+require 'fluent/plugin_helper/event_loop'
+require 'fluent/plugin_helper/timer'
+require 'fluent/plugin_helper/child_process'
+require 'fluent/plugin_helper/storage'
+require 'fluent/plugin_helper/parser'
+require 'fluent/plugin_helper/formatter'
+require 'fluent/plugin_helper/inject'
+require 'fluent/plugin_helper/extract'
+require 'fluent/plugin_helper/socket'
+require 'fluent/plugin_helper/server'
+require 'fluent/plugin_helper/retry_state'
+require 'fluent/plugin_helper/compat_parameters'
+
+module Fluent
+  module PluginHelper
+    module Mixin
+      def self.included(mod)
+        mod.extend(Fluent::PluginHelper)
+      end
+    end
+
+    def self.extended(mod)
+      def mod.inherited(subclass)
+        subclass.module_eval do
+          @_plugin_helpers_list = []
+        end
+      end
+    end
+
+    def helpers_internal(*snake_case_symbols)
+      helper_modules = []
+      snake_case_symbols.each do |name|
+        begin
+          helper_modules << Fluent::PluginHelper.const_get(name.to_s.split('_').map(&:capitalize).join)
+        rescue NameError
+          raise "Unknown plugin helper:#{name}"
+        end
+      end
+      include(*helper_modules)
+    end
+
+    def helpers(*snake_case_symbols)
+      @_plugin_helpers_list ||= []
+      @_plugin_helpers_list.concat(snake_case_symbols)
+      helpers_internal(*snake_case_symbols)
+    end
+
+    def plugin_helpers
+      @_plugin_helpers_list || []
+    end
+  end
+end

data/lib/fluent/plugin_helper/cert_option.rb

@@ -0,0 +1,159 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'openssl'
+require 'socket'
+
+# this module is only for Socket/Server plugin helpers
+module Fluent
+  module PluginHelper
+    module CertOption
+      def cert_option_create_context(version, insecure, ciphers, conf)
+        cert, key, extra = cert_option_server_validate!(conf)
+
+        ctx = OpenSSL::SSL::SSLContext.new(version)
+        unless insecure
+          # inject OpenSSL::SSL::SSLContext::DEFAULT_PARAMS
+          # https://bugs.ruby-lang.org/issues/9424
+          ctx.set_params({})
+
+          ctx.ciphers = ciphers
+        end
+
+        ctx.cert = cert
+        ctx.key = key
+        if extra && !extra.empty?
+          ctx.extra_chain_cert = extra
+        end
+
+        ctx
+      end
+
+      def cert_option_server_validate!(conf)
+        case
+        when conf.cert_path
+          raise Fluent::ConfigError, "private_key_path is required when cert_path is specified" unless conf.private_key_path
+          raise Fluent::ConfigError, "private_key_passphrase is required when cert_path is specified" unless conf.private_key_passphrase
+          cert_option_load(conf.cert_path, conf.private_key_path, conf.private_key_passphrase)
+
+        when conf.ca_cert_path
+          raise Fluent::ConfigError, "ca_private_key_path is required when ca_cert_path is specified" unless conf.ca_private_key_path
+          raise Fluent::ConfigError, "ca_private_key_passphrase is required when ca_cert_path is specified" unless conf.ca_private_key_passphrase
+          generate_opts = cert_option_cert_generation_opts_from_conf(conf)
+          cert_option_generate_server_pair_by_ca(
+            conf.ca_cert_path,
+            conf.ca_private_key_path,
+            conf.ca_private_key_passphrase,
+            generate_opts
+          )
+
+        when conf.insecure
+          log.warn "insecure TLS communication server is configured (using 'insecure' mode)"
+          generate_opts = cert_option_cert_generation_opts_from_conf(conf)
+          cert_option_generate_server_pair_self_signed(generate_opts)
+
+        else
+          raise Fluent::ConfigError, "no valid cert options configured. specify either 'cert_path', 'ca_cert_path' or 'insecure'"
+        end
+      end
+
+      def cert_option_load(cert_path, private_key_path, private_key_passphrase)
+        key = OpenSSL::PKey::RSA.new(File.read(private_key_path), private_key_passphrase)
+        certs = cert_option_certificates_from_file(cert_path)
+        cert = certs.shift
+        return cert, key, certs
+      end
+
+      def cert_option_cert_generation_opts_from_conf(conf)
+        {
+          private_key_length: conf.generate_private_key_length,
+          country: conf.generate_cert_country,
+          state: conf.generate_cert_state,
+          locality: conf.generate_cert_locality,
+          common_name: conf.generate_cert_common_name || ::Socket.gethostname,
+          expiration: conf.generate_cert_expiration,
+          digest: conf.generate_cert_digest,
+        }
+      end
+
+      def cert_option_generate_pair(opts, issuer = nil)
+        key = OpenSSL::PKey::RSA.generate(opts[:private_key_length])
+
+        subject = OpenSSL::X509::Name.new
+        subject.add_entry('C', opts[:country])
+        subject.add_entry('ST', opts[:state])
+        subject.add_entry('L', opts[:locality])
+        subject.add_entry('CN', opts[:common_name])
+
+        issuer ||= subject
+
+        cert = OpenSSL::X509::Certificate.new
+        cert.not_before = Time.at(0)
+        cert.not_after = Time.now + opts[:expiration]
+        cert.public_key = key
+        cert.serial = 1
+        cert.issuer = issuer
+        cert.subject  = subject
+
+        return cert, key
+      end
+
+      def cert_option_generate_ca_pair_self_signed(generate_opts)
+        cert, key = cert_option_generate_pair(generate_opts)
+
+        # basicConstraints: this cert is for CA or not
+        cert.add_extension OpenSSL::X509::Extension.new('basicConstraints', OpenSSL::ASN1.Sequence([OpenSSL::ASN1::Boolean(true)]))
+
+        cert.sign(key, generate_opts[:digest].to_s)
+        return cert, key
+      end
+
+      def cert_option_generate_server_pair_by_ca(ca_cert_path, ca_key_path, ca_key_passphrase, generate_opts)
+        ca_key = OpenSSL::PKey::RSA.new(File.read(ca_key_path), ca_key_passphrase)
+        ca_cert = OpenSSL::X509::Certificate.new(File.read(ca_cert_path))
+        cert, key = cert_option_generate_pair(generate_opts, ca_cert.subject)
+        raise "BUG: certificate digest algorithm not set" unless generate_opts[:digest]
+
+        # basicConstraints: this cert is for CA or not
+        cert.add_extension OpenSSL::X509::Extension.new('basicConstraints', OpenSSL::ASN1.Sequence([OpenSSL::ASN1::Boolean(false)]))
+        cert.add_extension OpenSSL::X509::Extension.new('nsCertType', 'server')
+
+        cert.sign(ca_key, generate_opts[:digest].to_s)
+        return cert, key, nil
+      end
+
+      def cert_option_generate_server_pair_self_signed(generate_opts)
+        cert, key = cert_option_generate_pair(generate_opts)
+        raise "BUG: certificate digest algorithm not set" unless generate_opts[:digest]
+
+        # basicConstraints: this cert is for CA or not
+        cert.add_extension OpenSSL::X509::Extension.new('basicConstraints', OpenSSL::ASN1.Sequence([OpenSSL::ASN1::Boolean(false)]))
+        cert.add_extension OpenSSL::X509::Extension.new('nsCertType', 'server')
+
+        cert.sign(key, generate_opts[:digest].to_s)
+        return cert, key, nil
+      end
+
+      def cert_option_certificates_from_file(path)
+        data = File.read(path)
+        pattern = Regexp.compile('-+BEGIN CERTIFICATE-+\n(?:[^-]*\n)+-+END CERTIFICATE-+\n', Regexp::MULTILINE)
+        list = []
+        data.scan(pattern){|match| list << OpenSSL::X509::Certificate.new(match) }
+        list
+      end
+    end
+  end
+end