fluent-plugin-openstack 0.0.2

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 9b33942f008ad19c796f4454ba2e9176ab90dc7f43e974f9dda81d6201a4ec3d
+  data.tar.gz: c2414c02db5bebd37f5ed9103a9a25f9f836a5de3037002580fb7ee5f399cf6b
+SHA512:
+  metadata.gz: 2ac64444f46c298403798f7c74fc4f92d52cd4b57b0ee4f214e5dd2f4cc1898c1344f9564802b54c5c1c11eff644c16fe9f4e1b38b339d8aebbcc65dbb99e2e2
+  data.tar.gz: 1d9b1df6f12088f51d7aed52fd08aa0b7a2a847535cea83439551fad4904ab9ad2b893f613184a7574f1ec1d662e395c32b8e095df57a211a71dabfbbfc27691

data/.gitignore

@@ -0,0 +1,13 @@
+~*
+#*
+*~
+[._]*.s[a-w][a-z]
+.DS_Store
+
+*.gem
+.bundle
+Gemfile.lock
+vendor
+.ruby-version
+
+test/tmp/

data/Gemfile

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+source 'https://rubygems.org'
+
+gemspec

data/LICENSE.md

@@ -0,0 +1,7 @@
+Copyright 2019 robert.krzysztoforski@protonmail.com
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,6 @@
+## OpenStack Storage Service (Swift) plugin for Fluent
+
+### Overview
+
+This gem is based on hard work of: `https://github.com/yuuzi41/fluent-plugin-openstack`.
+It is simplified and refactored version of this gem.

data/Rakefile

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+require 'bundler/gem_tasks'
+Bundler::GemHelper.install_tasks
+
+require 'rake/testtask'
+
+Rake::TestTask.new(:test) do |t|
+  t.libs.push('lib', 'test')
+  t.test_files = FileList['test/**/test_*.rb']
+  t.verbose = true
+  t.warning = false
+end
+
+task default: [:test]

data/VERSION

@@ -0,0 +1 @@
+0.0.2

data/fluent-plugin-openstack.gemspec

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+$LOAD_PATH.push File.expand_path('lib', __dir__)
+
+Gem::Specification.new do |gem|
+  gem.name        = 'fluent-plugin-openstack'
+  gem.description = 'OpenStack Storage Service (Swift) plugin for Fluentd'
+  gem.homepage    = 'https://github.com/jnsolutions/fluent-plugin-openstack'
+  gem.summary     = gem.description
+  gem.version     = File.read('VERSION').strip
+  gem.license     = 'MIT'
+  gem.authors     = ['brissenden']
+  gem.email       = 'robert.krzysztoforski@protonmail.com'
+  gem.platform    = Gem::Platform::RUBY
+  gem.files       = `git ls-files`.split("\n")
+  gem.test_files  = `git ls-files -- {test,spec,features}/*`.split("\n")
+  gem.executables = `git ls-files -- bin/*`.split("\n").map { |f| File.basename(f) }
+  gem.require_paths = ['lib']
+
+  gem.add_runtime_dependency 'fluentd', ['>= 0.14.2', '< 2']
+  gem.add_runtime_dependency 'fog-openstack'
+  gem.add_runtime_dependency 'uuidtools'
+
+  gem.add_development_dependency 'bundler', '~> 2.0.2'
+  gem.add_development_dependency 'flexmock', '>= 1.2.0'
+  gem.add_development_dependency 'rake', '~> 12.0'
+  gem.add_development_dependency 'test-unit', '>= 3.1.0'
+
+  gem.add_dependency('xmlrpc') if RUBY_VERSION.to_s >= '2.4'
+end

data/lib/fluent/plugin/out_swift.rb

@@ -0,0 +1,314 @@
+# frozen_string_literal: true
+
+require 'fluent/plugin/output'
+require 'fluent/timezone'
+require 'fog/openstack'
+require 'zlib'
+require 'time'
+require 'tempfile'
+require 'open3'
+
+module Fluent::Plugin
+  class SwiftOutput < Output
+    Fluent::Plugin.register_output('swift', self)
+
+    DEFAULT_FORMAT_TYPE = 'out_file'
+    MAX_HEX_RANDOM_LENGTH = 16
+
+    desc 'Path prefix of the files on Swift'
+    config_param :path, :string, default: ''
+    # OpenStack AUTH
+    desc "Authentication URL. Set a value or `#{ENV['OS_AUTH_URL']}`"
+    config_param :auth_url, :string
+    desc "Authentication User Name. If you use TempAuth, auth_user is ACCOUNT:USER. Set a value or use `#{ENV['OS_USERNAME']}`"
+    config_param :auth_user, :string
+    desc "Authentication Key (Password). Set a value or use `#{ENV['OS_PASSWORD']}`"
+    config_param :auth_api_key, :string
+    # Identity v2
+    config_param :auth_tenant, :string, default: nil
+    # Identity v3
+    desc "Authentication Project. Set a value or use `#{ENV['OS_PROJECT_NAME']}`"
+    config_param :project_name, :string, default: nil
+    desc "Authentication Domain. Set a value or use `#{ENV['OS_PROJECT_DOMAIN_NAME']}`"
+    config_param :domain_name, :string, default: nil
+    desc "Authentication Region. Optional, not required if there is only one region available. Set a value or use `#{ENV['OS_REGION_NAME']}`"
+    config_param :auth_region, :string, default: nil
+    config_param :swift_account, :string, default: nil
+
+    desc 'Swift container name'
+    config_param :swift_container, :string
+    desc 'Archive format on Swift'
+    config_param :store_as, :string, default: 'gzip'
+    desc 'If false, the certificate of endpoint will not be verified'
+    config_param :ssl_verify, :bool, default: true
+    desc 'The format of Swift object keys'
+    config_param :swift_object_key_format, :string, default: '%{path}%{time_slice}_%{index}.%{file_extension}'
+    desc 'Create Swift container if it does not exists'
+    config_param :auto_create_container, :bool, default: true
+    config_param :check_apikey_on_start, :bool, default: true
+    desc 'URI of proxy environment'
+    config_param :proxy_uri, :string, default: nil
+    desc 'The length of `%{hex_random}` placeholder(4-16)'
+    config_param :hex_random_length, :integer, default: 4
+    desc '`sprintf` format for `%{index}`'
+    config_param :index_format, :string, default: '%d'
+    desc 'Overwrite already existing path'
+    config_param :overwrite, :bool, default: false
+
+    config_section :format do
+      config_set_default :@type, DEFAULT_FORMAT_TYPE
+    end
+
+    config_section :buffer do
+      config_set_default :chunk_keys, ['time']
+      config_set_default :timekey, (60 * 60 * 24)
+    end
+
+    helpers :compat_parameters, :formatter, :inject
+
+    def initialize
+      super
+      self.uuid_flush_enabled = false
+    end
+
+    def configure(config)
+      compat_parameters_convert(config, :buffer, :formatter, :inject)
+
+      super
+
+      if auth_url.blank?
+        raise Fluent::ConfigError, 'auth_url parameter or OS_AUTH_URL variable not defined'
+      end
+      if auth_user.blank?
+        raise Fluent::ConfigError, 'auth_user parameter or OS_USERNAME variable not defined'
+      end
+      if auth_api_key.blank?
+        raise Fluent::ConfigError, 'auth_api_key parameter or OS_PASSWORD variable not defined'
+      end
+
+      if project_name.blank?
+        raise Fluent::ConfigError, 'project_name parameter or OS_PROJECT_NAME variable not defined'
+      end
+      if domain_name.blank?
+        raise Fluent::ConfigError, 'domain_name parameter or OS_PROJECT_DOMAIN_NAME variable not defined'
+      end
+
+      self.ext, self.mime_type = case store_as
+                                 when 'gzip' then ['gz', 'application/x-gzip']
+                                 when 'lzo' then
+                                   begin
+                                     Open3.capture3('lzop -V')
+                                   rescue Errno::ENOENT
+                                     raise ConfigError, "'lzop' utility must be in PATH for LZO compression"
+                                   end
+                                   ['lzo', 'application/x-lzop']
+                                 when 'json' then ['json', 'application/json']
+                                 else ['txt', 'text/plain']
+      end
+
+      self.formatter = formatter_create
+
+      if hex_random_length > MAX_HEX_RANDOM_LENGTH
+        raise Fluent::ConfigError, "hex_random_length parameter must be less than or equal to #{MAX_HEX_RANDOM_LENGTH}"
+      end
+
+      unless index_format =~ /^%(0\d*)?[dxX]$/
+        raise Fluent::ConfigError, 'index_format parameter should follow `%[flags][width]type`. `0` is the only supported flag, and is mandatory if width is specified. `d`, `x` and `X` are supported types'
+      end
+
+      self.swift_object_key_format = configure_swift_object_key_format
+      # For backward compatibility
+      # TODO: Remove time_slice_format when end of support compat_parameters
+      self.values_for_swift_object_chunk = {}
+      self.time_slice_with_tz = Fluent::Timezone.formatter(timekey_zone, config['time_slice_format'] || timekey_to_timeformat(timekey: buffer_config['timekey']))
+    end
+
+    def multi_workers_ready?
+      true
+    end
+
+    def start
+      Excon.defaults[:ssl_verify_peer] = ssl_verify
+      begin
+        self.storage = Fog::Storage.new(
+          provider: 'OpenStack',
+          openstack_auth_url: auth_url,
+          openstack_username: auth_user,
+          openstack_api_key: auth_api_key,
+          openstack_tenant: auth_tenant,
+          openstack_project_name: project_name,
+          openstack_domain_name: domain_name,
+          openstack_region: auth_region
+        )
+      rescue StandardError => exp
+        raise "Can't call Swift API. Please check your ENV OS_*, your credentials or auth_url configuration. Error: #{exp.inspect}"
+      end
+      if swift_account
+        storage.change_account(swift_account)
+      end
+      check_container
+      super
+    end
+
+    # TODO: dead method?
+    def format(tag, time, record)
+      r = inject_values_to_record(tag, time, record)
+      formatter.format(tag, time, r)
+    end
+
+    def write(chunk)
+      i = 0
+      metadata = chunk.metadata
+      previous_path = nil
+      time_slice = if metadata.timekey.nil?
+                     ''
+                   else
+                     time_slice_with_tz.call(metadata.timekey)
+                   end
+
+      while check_object_exists(object: swift_path)
+        values_for_swift_object_chunk[chunk.unique_id] ||= {
+          '%{hex_random}' => hex_random(chunk: chunk)
+        }
+        values_for_swift_object_key_pre = {
+          '%{path}' => path,
+          '%{file_extension}' => ext
+        }
+        values_for_swift_object_key_post = {
+          '%{time_slice}' => time_slice,
+          '%{index}' => format(index_format, i)
+        }.merge!(values_for_swift_object_chunk[chunk.unique_id])
+        values_for_swift_object_key_post['%{uuid_flush}'] = uuid_random if uuid_flush_enabled
+
+        swift_path = swift_object_key_format.gsub(/%{[^}]+}/) do |matched_key|
+          values_for_swift_object_key_pre.fetch(matched_key, matched_key)
+        end
+
+        swift_path = extract_placeholders(swift_path, metadata)
+        swift_path = swift_path.gsub(/%{[^}]+}/, values_for_swift_object_key_post)
+        if (i > 0) && (swift_path == previous_path)
+          if overwrite
+            log.warn "#{swift_path} already exists, but will overwrite"
+            break
+          else
+            raise "duplicated path is generated. use %{index} in swift_object_key_format: path = #{swift_path}"
+          end
+        end
+
+        i += 1
+        previous_path = swift_path
+      end
+
+      tmp = Tempfile.new('swift-')
+      tmp.binmode
+      begin
+        if store_as == 'gzip'
+          w = Zlib::GzipWriter.new(tmp)
+          chunk.write_to(w)
+          w.close
+        elsif store_as == 'lzo'
+          w = Tempfile.new('chunk-tmp')
+          chunk.write_to(w)
+          w.close
+          tmp.close
+          system "lzop -qf1 -o #{tmp.path} #{w.path}"
+        else
+          chunk.write_to(tmp)
+          tmp.close
+        end
+        File.open(tmp.path) do |file|
+          storage.put_object(swift_container, swift_path, file, content_type: mime_type)
+          values_for_swift_object_chunk.delete(chunk.unique_id)
+        end
+      ensure
+        begin
+          tmp.close(true)
+        rescue StandardError
+          nil
+        end
+        begin
+          w.close
+        rescue StandardError
+          nil
+        end
+        begin
+          w.unlink
+        rescue StandardError
+          nil
+        end
+      end
+    end
+
+    private
+
+    attr_accessor :uuid_flush_enabled, :storage, :ext, :mime_type, :formatter, :values_for_swift_object_chunk, :time_slice_with_tz
+
+    def hex_random(chunk:)
+      unique_hex = Fluent::UniqueId.hex(chunk.unique_id)
+      unique_hex.reverse!
+      unique_hex[0...hex_random_length]
+    end
+
+    def uuid_random
+      ::UUIDTools::UUID.random_create.to_s
+    end
+
+    # This is stolen from Fluentd
+    def timekey_to_timeformat(timekey:)
+      case timekey
+      when nil          then ''
+      when 0...60       then '%Y%m%d%H%M%S'
+      when 60...3600    then '%Y%m%d%H%M'
+      when 3600...86_400 then '%Y%m%d%H'
+      else '%Y%m%d'
+      end
+    end
+
+    def check_container
+      storage.get_container(swift_container)
+    rescue Fog::OpenStack::Storage::NotFound
+      if auto_create_container
+        $log.info "Creating container #{swift_container} on #{auth_url}, #{swift_account}"
+        storage.put_container(swift_container)
+      else
+        raise "The specified container does not exist: container = #{swift_container}"
+      end
+    end
+
+    def configure_swift_object_key_format
+      %w[%{uuid} %{uuid:random} %{uuid:hostname} %{uuid:timestamp}].each do |ph|
+        if swift_object_key_format.include?(ph)
+          raise Fluent::ConfigError, %(#{ph} placeholder in swift_object_key_format is removed)
+        end
+      end
+
+      if swift_object_key_format.include?('%{uuid_flush}')
+        begin
+          require 'uuidtools'
+        rescue LoadError
+          raise Fluent::ConfigError, 'uuidtools gem not found. Install uuidtools gem first'
+        end
+        begin
+          uuid_random
+        rescue StandardError => e
+          raise Fluent::ConfigError, "Generating uuid doesn't work. Can't use %{uuid_flush} on this environment. #{e}"
+        end
+        self.uuid_flush_enabled = true
+      end
+
+      swift_object_key_format.gsub('%{hostname}') do |_expr|
+        log.warn "%{hostname} will be removed in the future. Use \"\#{Socket.gethostname}\" instead"
+        Socket.gethostname
+      end
+    end
+
+    def check_object_exists(object:)
+      begin
+        storage.head_object(swift_container, object)
+        true
+      rescue Fog::OpenStack::Storage::NotFound
+        false
+      end
+    end
+  end
+end

data/test/helper.rb

@@ -0,0 +1,10 @@
+# frozen_string_literal: true
+
+$LOAD_PATH.unshift(File.expand_path('..', __dir__))
+require 'test-unit'
+require 'fluent/test'
+require 'fluent/test/driver/output'
+require 'fluent/test/helpers'
+
+Test::Unit::TestCase.include(Fluent::Test::Helpers)
+Test::Unit::TestCase.extend(Fluent::Test::Helpers)

data/test/plugin/test_out_swift.rb

@@ -0,0 +1,80 @@
+# frozen_string_literal: true
+
+require 'helper'
+require 'fluent/plugin/out_swift'
+
+class SwiftOutputTest < Test::Unit::TestCase
+  setup do
+    Fluent::Test.setup
+  end
+
+  def teardown
+    Dir.glob('test/tmp/*').each { |file| FileUtils.rm_f(file) }
+  end
+
+  CONFIG_NONE = %(
+    swift_container CONTAINER_NAME
+  )
+
+  CONFIG_ENV = %(
+    swift_container CONTAINER_NAME
+    auth_url "#{ENV['EMPTY_OS_AUTH_URL']}"
+    auth_user test:tester
+    auth_api_key testing
+  )
+
+  CONFIG = %(
+    auth_url https://127.0.0.1/auth/v3
+    auth_user test:tester
+    auth_api_key testing
+    domain_name default
+    project_name test_project
+    auth_region RegionOne
+    swift_container CONTAINER_NAME
+    path logs/
+    swift_object_key_format %{path}%{time_slice}_%{index}.%{file_extension}
+    ssl_verify false
+    buffer_path /var/log/fluent/swift
+    buffer_type memory
+    time_slice_format %Y%m%d-%H
+    time_slice_wait 10m
+    utc
+  )
+
+  def create_driver(conf = CONFIG)
+    Fluent::Test::Driver::Output.new(Fluent::Plugin::SwiftOutput) do
+      def format(tag, time, record)
+        super
+      end
+
+      def write(chunk)
+        chunk.read
+      end
+
+      private
+
+      def check_container; end
+    end.configure(conf)
+  end
+
+  def test_auth_url_absent
+    assert_raise_message(/'auth_url' parameter is required/) do
+      create_driver(CONFIG_NONE)
+    end
+  end
+
+  def test_auth_url_empty
+    assert_raise_message(/auth_url parameter or OS_AUTH_URL variable not defined/) do
+      create_driver(CONFIG_ENV)
+    end
+  end
+
+  def test_configure
+    d = create_driver(CONFIG)
+    assert_equal 'test:tester', d.instance.auth_user
+    assert_equal 'testing', d.instance.auth_api_key
+    assert_equal 'RegionOne', d.instance.auth_region
+    assert_equal 'CONTAINER_NAME', d.instance.swift_container
+    assert_equal 'logs/', d.instance.path
+  end
+end

metadata

@@ -0,0 +1,172 @@
+--- !ruby/object:Gem::Specification
+name: fluent-plugin-openstack
+version: !ruby/object:Gem::Version
+  version: 0.0.2
+platform: ruby
+authors:
+- brissenden
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2019-08-28 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: fluentd
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.14.2
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '2'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.14.2
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '2'
+- !ruby/object:Gem::Dependency
+  name: fog-openstack
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: uuidtools
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 2.0.2
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 2.0.2
+- !ruby/object:Gem::Dependency
+  name: flexmock
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.2.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.2.0
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '12.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '12.0'
+- !ruby/object:Gem::Dependency
+  name: test-unit
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 3.1.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 3.1.0
+- !ruby/object:Gem::Dependency
+  name: xmlrpc
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: OpenStack Storage Service (Swift) plugin for Fluentd
+email: robert.krzysztoforski@protonmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- Gemfile
+- LICENSE.md
+- README.md
+- Rakefile
+- VERSION
+- fluent-plugin-openstack.gemspec
+- lib/fluent/plugin/out_swift.rb
+- test/helper.rb
+- test/plugin/test_out_swift.rb
+homepage: https://github.com/jnsolutions/fluent-plugin-openstack
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.0.6
+signing_key: 
+specification_version: 4
+summary: OpenStack Storage Service (Swift) plugin for Fluentd
+test_files:
+- test/helper.rb
+- test/plugin/test_out_swift.rb