fluent-plugin-openstack 1.0.0 → 1.0.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: e3404369fda531301efc4a6f339d65470f8f77e0d18358dc814925384b555323
-  data.tar.gz: b7ee470932b4c0b148ac1fa9f55a2df8ae9987b5c6057e1a352b20e85ba1557a
+  metadata.gz: 4b27ffca51d8e3d40e7793106d6c94a6a2cc13db511aec390cdf6755536b469f
+  data.tar.gz: d79d1b950a0846e60c05a9b56ddff6b3d4f2aea2a195f05ab821cafc601daacd
 SHA512:
-  metadata.gz: 4e5bba22105167186f6597e0e7c0dd1ed75a5d0d55b85968898c3c29b3a8fbcc4b676f5b6450e45587f2228ccd2668572ccf987c1b60a3930eb18d0571474fb4
-  data.tar.gz: 2036ccd265066d9d86b64463f58f767e656ff8d053afa3f5abb18a3063237f54f39e6b08763339dccd9ac0938b4257bcb61c4dd03643471e2691aef55f26ace8
+  metadata.gz: 25daa97506ab9150f10b26a54bceb495bc88552a12c505076f365465a4c3ec6e4108a8651bec2bfe1ba213b503af5820cdafe16b070a2edb1b371df618fb8d8c
+  data.tar.gz: cd25abdefd40c317246e653ef3d41909c8aea18eefd8c5828d815af97aba61e4302d49bedb4ff75eec0511523a27b63b066788fc1c68e975f7f8db1536563721

data/VERSION

@@ -1 +1 @@
-1.0.0
+1.0.1

data/lib/fluent/plugin/out_swift.rb

@@ -16,7 +16,7 @@ module Fluent::Plugin
     MAX_HEX_RANDOM_LENGTH = 16
 
     desc 'Path prefix of the files on Swift'
-    config_param :path, :string, default: ''
+    config_param :path, :string, default: '%Y%m%d'
     # OpenStack AUTH
     desc "Authentication URL. Set a value or `#{ENV['OS_AUTH_URL']}`"
     config_param :auth_url, :string
@@ -36,13 +36,13 @@ module Fluent::Plugin
     desc 'If false, the certificate of endpoint will not be verified'
     config_param :ssl_verify, :bool, default: true
     desc 'The format of Swift object keys'
-    config_param :swift_object_key_format, :string, default: '%{path}%{time_slice}_%{index}.%{file_extension}'
+    config_param :swift_object_key_format, :string, default: '%{path}/%{time_slice}_%{index}.%{file_extension}'
     desc 'Create Swift container if it does not exists'
     config_param :auto_create_container, :bool, default: true
     config_param :check_apikey_on_start, :bool, default: true
     desc 'URI of proxy environment'
     config_param :proxy_uri, :string, default: nil
-    desc 'The length of `%{hex_random}` placeholder(4-16)'
+    desc 'The length of `%{hex_random}` placeholder (4 - 16)'
     config_param :hex_random_length, :integer, default: 4
     desc '`sprintf` format for `%{index}`'
     config_param :index_format, :string, default: '%d'
@@ -70,6 +70,8 @@ module Fluent::Plugin
 
       super
 
+      $log.info("config: #{config}")
+
       if auth_url.empty?
         raise Fluent::ConfigError, 'auth_url parameter or OS_AUTH_URL variable not defined'
       end
@@ -80,6 +82,14 @@ module Fluent::Plugin
         raise Fluent::ConfigError, 'auth_api_key parameter or OS_PASSWORD variable not defined'
       end
 
+      if hex_random_length > MAX_HEX_RANDOM_LENGTH
+        raise Fluent::ConfigError, "hex_random_length parameter must be less than or equal to #{MAX_HEX_RANDOM_LENGTH}"
+      end
+
+      unless index_format =~ /^%(0\d*)?[dxX]$/
+        raise Fluent::ConfigError, 'index_format parameter should follow `%[flags][width]type`. `0` is the only supported flag, and is mandatory if width is specified. `d`, `x` and `X` are supported types'
+      end
+
       self.ext, self.mime_type = case store_as
                                  when 'gzip' then ['gz', 'application/x-gzip']
                                  when 'lzo' then
@@ -91,23 +101,14 @@ module Fluent::Plugin
                                    ['lzo', 'application/x-lzop']
                                  when 'json' then ['json', 'application/json']
                                  else ['txt', 'text/plain']
-      end
+                                 end
 
       self.formatter = formatter_create
-
-      if hex_random_length > MAX_HEX_RANDOM_LENGTH
-        raise Fluent::ConfigError, "hex_random_length parameter must be less than or equal to #{MAX_HEX_RANDOM_LENGTH}"
-      end
-
-      unless index_format =~ /^%(0\d*)?[dxX]$/
-        raise Fluent::ConfigError, 'index_format parameter should follow `%[flags][width]type`. `0` is the only supported flag, and is mandatory if width is specified. `d`, `x` and `X` are supported types'
-      end
-
       self.swift_object_key_format = configure_swift_object_key_format
       # For backward compatibility
       # TODO: Remove time_slice_format when end of support compat_parameters
       self.values_for_swift_object_chunk = {}
-      self.time_slice_with_tz = Fluent::Timezone.formatter(timekey_zone, config['time_slice_format'] || timekey_to_timeformat(timekey: buffer_config['timekey']))
+      self.time_slice_with_tz = Fluent::Timezone.formatter(timekey_zone, config['time_slice_format'])
     end
 
     def multi_workers_ready?
@@ -145,10 +146,15 @@ module Fluent::Plugin
       time_slice = if metadata.timekey.nil?
                      ''
                    else
+                     $log.info("timekey: #{metadata.timekey}")
+                     $log.info("metadata: #{metadata}")
                      time_slice_with_tz.call(metadata.timekey)
                    end
 
       begin
+        $log.info("time_slice: #{time_slice}")
+        $log.info("index_format: #{index_format}")
+
         values_for_swift_object_chunk[chunk.unique_id] ||= {
           '%{hex_random}' => hex_random(chunk: chunk)
         }
@@ -158,9 +164,11 @@ module Fluent::Plugin
         }
         values_for_swift_object_key_post = {
           '%{time_slice}' => time_slice,
-          '%{index}' => sprintf(index_format, i)
+          '%{index}' => format(index_format, i)
         }.merge!(values_for_swift_object_chunk[chunk.unique_id])
 
+        $log.info("values_for_swift_object_key_post: #{values_for_swift_object_key_post}")
+
         if uuid_flush_enabled
           values_for_swift_object_key_post['%{uuid_flush}'] = uuid_random
         end
@@ -169,11 +177,18 @@ module Fluent::Plugin
           values_for_swift_object_key_pre.fetch(matched_key, matched_key)
         end
 
+        $log.info("swift_path 1: #{swift_path}")
+
         swift_path = extract_placeholders(swift_path, metadata)
+
+        $log.info("swift_path 2: #{swift_path}")
+
         swift_path = swift_path.gsub(/%{[^}]+}/, values_for_swift_object_key_post)
+
+        $log.info("swift_path 3: #{swift_path}")
         if i.positive? && (swift_path == previous_path)
           if overwrite
-            log.warn "File: #{swift_path} already exists, but will overwrite!"
+            log.warn("File: #{swift_path} already exists, but will overwrite!")
             break
           else
             raise "Duplicated path is generated. Use %{index} in swift_object_key_format: Path: #{swift_path}"
@@ -202,7 +217,12 @@ module Fluent::Plugin
           tmp.close
         end
         File.open(tmp.path) do |file|
-          storage.put_object(swift_container, swift_path, file, content_type: mime_type)
+          storage.put_object(
+            swift_container,
+            swift_path,
+            file,
+            content_type: mime_type
+          )
           values_for_swift_object_chunk.delete(chunk.unique_id)
         end
       ensure
@@ -226,7 +246,13 @@ module Fluent::Plugin
 
     private
 
-    attr_accessor :uuid_flush_enabled, :storage, :ext, :mime_type, :formatter, :values_for_swift_object_chunk, :time_slice_with_tz
+    attr_accessor :uuid_flush_enabled,
+                  :storage,
+                  :ext,
+                  :mime_type,
+                  :formatter,
+                  :values_for_swift_object_chunk,
+                  :time_slice_with_tz
 
     def hex_random(chunk:)
       unique_hex = Fluent::UniqueId.hex(chunk.unique_id)
@@ -238,25 +264,14 @@ module Fluent::Plugin
       ::UUIDTools::UUID.random_create.to_s
     end
 
-    # This is stolen from Fluentd
-    def timekey_to_timeformat(timekey:)
-      case timekey
-      when nil          then ''
-      when 0...60       then '%Y%m%d%H%M%S'
-      when 60...3600    then '%Y%m%d%H%M'
-      when 3600...86_400 then '%Y%m%d%H'
-      else '%Y%m%d'
-      end
-    end
-
     def check_container
       storage.get_container(swift_container)
     rescue Fog::OpenStack::Storage::NotFound
       if auto_create_container
-        $log.info "Creating container #{swift_container} on #{auth_url}, #{swift_account}"
+        $log.info("Creating container `#{swift_container}` on `#{auth_url}`, `#{swift_account}`.")
         storage.put_container(swift_container)
       else
-        raise "The specified container does not exist: container = #{swift_container}"
+        raise "The specified container does not exist: #{swift_container}."
       end
     end
 
@@ -282,7 +297,7 @@ module Fluent::Plugin
       end
 
       swift_object_key_format.gsub('%{hostname}') do |_expr|
-        log.warn "%{hostname} will be removed in the future. Use \"\#{Socket.gethostname}\" instead"
+        log.warn("%{hostname} will be removed in the future. Use `#{Socket.gethostname}` instead.")
         Socket.gethostname
       end
     end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-openstack
 version: !ruby/object:Gem::Version
-  version: 1.0.0
+  version: 1.0.1
 platform: ruby
 authors:
 - brissenden
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-08-28 00:00:00.000000000 Z
+date: 2019-09-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd