fluentd 1.13.3 → 1.14.2

data/lib/fluent/plugin/metrics.rb

@@ -0,0 +1,119 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'socket'
+
+require 'fluent/plugin/base'
+
+require 'fluent/log'
+require 'fluent/unique_id'
+require 'fluent/plugin_id'
+
+module Fluent
+  module Plugin
+    class Metrics < Base
+      include PluginId
+      include PluginLoggerMixin
+      include UniqueId::Mixin
+
+      DEFAULT_TYPE = 'local'
+
+      configured_in :metrics
+
+      config_param :default_labels, :hash, default: {agent: "Fluentd", hostname: "#{Socket.gethostname}"}
+      config_param :labels, :hash, default: {}
+
+      attr_reader :use_gauge_metric
+      attr_reader :has_methods_for_gauge, :has_methods_for_counter
+
+      def initialize
+        super
+
+        @has_methods_for_counter = false
+        @has_methods_for_gauge = false
+        @use_gauge_metric = false
+      end
+
+      def configure(conf)
+        super
+
+        if use_gauge_metric
+          @has_methods_for_gauge = has_methods_for_gauge?
+        else
+          @has_methods_for_counter = has_methods_for_counter?
+        end
+      end
+
+      # Some metrics should be counted by gauge.
+      # ref: https://prometheus.io/docs/concepts/metric_types/#gauge
+      def use_gauge_metric=(use_gauge_metric=false)
+        @use_gauge_metric = use_gauge_metric
+      end
+
+      def create(namespace:, subsystem:,name:,help_text:,labels: {})
+        # This API is for cmetrics type.
+      end
+
+      def get
+        raise NotImplementedError, "Implement this method in child class"
+      end
+
+      def inc
+        raise NotImplementedError, "Implement this method in child class"
+      end
+
+      def dec
+        raise NotImplementedError, "Implement this method in child class"
+      end
+
+      def add(value)
+        raise NotImplementedError, "Implement this method in child class"
+      end
+
+      def sub(value)
+        raise NotImplementedError, "Implement this method in child class"
+      end
+
+      def set(value)
+        raise NotImplementedError, "Implement this method in child class"
+      end
+
+      private
+
+      def has_methods_for_counter?
+        implemented_methods = self.class.instance_methods(false)
+
+        if [:get, :inc, :add].all? {|e| implemented_methods.include?(e)} &&
+           [:set].all?{|e| self.class.method_defined?(e)}
+          true
+        else
+          raise "BUG: metrics plugin on counter mode MUST implement `get`, `inc`, `add` methods. And aliased `set` methods should be aliased from another method"
+        end
+      end
+
+      def has_methods_for_gauge?
+        implemented_methods = self.class.instance_methods(false)
+
+        if [:get, :inc, :add].all? {|e| implemented_methods.include?(e)} &&
+           [:set, :dec, :sub].all?{|e| self.class.method_defined?(e)}
+          true
+        else
+          raise "BUG: metrics plugin on gauge mode MUST implement `get`, `inc`, and `add` methods. And `dec`, `sub`, and `set` methods should be aliased from other methods"
+        end
+      end
+    end
+  end
+end

data/lib/fluent/plugin/metrics_local.rb

@@ -0,0 +1,96 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/plugin'
+require 'fluent/plugin/metrics'
+
+module Fluent
+  module Plugin
+    class LocalMetrics < Metrics
+      Fluent::Plugin.register_metrics('local', self)
+
+      def initialize
+        super
+        @store = 0
+        @monitor = Monitor.new
+      end
+
+      def configure(conf)
+        super
+
+        if use_gauge_metric
+          class << self
+            alias_method :dec, :dec_gauge
+            alias_method :set, :set_gauge
+            alias_method :sub, :sub_gauge
+          end
+        else
+          class << self
+            alias_method :set, :set_counter
+          end
+        end
+      end
+
+      def multi_workers_ready?
+        true
+      end
+
+      def get
+        @monitor.synchronize do
+          @store
+        end
+      end
+
+      def inc
+        @monitor.synchronize do
+          @store += 1
+        end
+      end
+
+      def dec_gauge
+        @monitor.synchronize do
+          @store -= 1
+        end
+      end
+
+      def add(value)
+        @monitor.synchronize do
+          @store += value
+        end
+      end
+
+      def sub_gauge(value)
+        @monitor.synchronize do
+          @store -= value
+        end
+      end
+
+      def set_counter(value)
+        return if @store > value
+
+        @monitor.synchronize do
+          @store = value
+        end
+      end
+
+      def set_gauge(value)
+        @monitor.synchronize do
+          @store = value
+        end
+      end
+    end
+  end
+end

data/lib/fluent/plugin/multi_output.rb

@@ -27,6 +27,7 @@ module Fluent
       include PluginHelper::Mixin # for event_emitter
 
       helpers :event_emitter # to get router from agent, which will be supplied to child plugins
+      helpers_internal :metrics
 
       config_section :store, param_name: :stores, multi: true, required: true do
         config_argument :arg, :string, default: ''
@@ -46,11 +47,40 @@ module Fluent
 
         @counter_mutex = Mutex.new
         # TODO: well organized counters
-        @num_errors = 0
-        @emit_count = 0
-        @emit_records = 0
+        @num_errors_metrics = nil
+        @emit_count_metrics = nil
+        @emit_records_metrics = nil
+        @emit_size_metrics = nil
         # @write_count = 0
         # @rollback_count = 0
+        @enable_size_metrics = false
+      end
+
+      def num_errors
+        @num_errors_metrics.get
+      end
+
+      def emit_count
+        @emit_count_metrics.get
+      end
+
+      def emit_size
+        @emit_size_metrics.get
+      end
+
+      def emit_records
+        @emit_records_metrics.get
+      end
+
+      def statistics
+        stats = {
+          'num_errors' => @num_errors_metrics.get,
+          'emit_records' => @emit_records_metrics.get,
+          'emit_count' => @emit_count_metrics.get,
+          'emit_size' => @emit_size_metrics.get,
+        }
+
+        { 'multi_output' => stats }
       end
 
       def multi_output?
@@ -60,6 +90,12 @@ module Fluent
       def configure(conf)
         super
 
+        @num_errors_metrics = metrics_create(namespace: "fluentd", subsystem: "multi_output", name: "num_errors", help_text: "Number of count num errors")
+        @emit_count_metrics = metrics_create(namespace: "fluentd", subsystem: "multi_output", name: "emit_records", help_text: "Number of count emits")
+        @emit_records_metrics = metrics_create(namespace: "fluentd", subsystem: "multi_output", name: "emit_records", help_text: "Number of emit records")
+        @emit_size_metrics =  metrics_create(namespace: "fluentd", subsystem: "multi_output", name: "emit_size", help_text: "Total size of emit events")
+        @enable_size_metrics = !!system_config.enable_size_metrics
+
         @stores.each do |store|
           store_conf = store.corresponding_config_element
           type = store_conf['@type']
@@ -143,12 +179,13 @@ module Fluent
       end
 
       def emit_sync(tag, es)
-        @counter_mutex.synchronize{ @emit_count += 1 }
+        @emit_count_metrics.inc
         begin
           process(tag, es)
-          @counter_mutex.synchronize{ @emit_records += es.size }
+          @emit_records_metrics.add(es.size)
+          @emit_size_metrics.add(es.to_msgpack_stream.bytesize) if @enable_size_metrics
         rescue
-          @counter_mutex.synchronize{ @num_errors += 1 }
+          @num_errors_metrics.inc
           raise
         end
       end

data/lib/fluent/plugin/out_copy.rb

@@ -46,7 +46,7 @@ module Fluent::Plugin
         @ignore_errors << (store.arg.include?('ignore_error'))
         @ignore_if_prev_successes << (store.arg.include?('ignore_if_prev_success'))
       }
-      if @ignore_errors.uniq.size == 1 && @ignore_errors.include?(true) && @ignore_if_prev_successes.include?(false)
+      if @ignore_errors.uniq.size == 1 && @ignore_errors.include?(true) && !@ignore_if_prev_successes.include?(true)
         log.warn "ignore_errors are specified in all <store>, but ignore_if_prev_success is not specified. Is this intended?"
       end
     end

data/lib/fluent/plugin/out_forward.rb

@@ -167,6 +167,8 @@ module Fluent::Plugin
       @usock = nil
       @keep_alive_watcher_interval = 5 # TODO
       @suspend_flush = false
+      @healthy_nodes_count_metrics = nil
+      @registered_nodes_count_metrics = nil
     end
 
     def configure(conf)
@@ -265,6 +267,9 @@ module Fluent::Plugin
       end
 
       raise Fluent::ConfigError, "ack_response_timeout must be a positive integer" if @ack_response_timeout < 1
+      @healthy_nodes_count_metrics = metrics_create(namespace: "fluentd", subsystem: "output", name: "healthy_nodes_count", help_text: "Number of count healthy nodes", prefer_gauge: true)
+      @registered_nodes_count_metrics = metrics_create(namespace: "fluentd", subsystem: "output", name: "registered_nodes_count", help_text: "Number of count registered nodes", prefer_gauge: true)
+
     end
 
     def multi_workers_ready?
@@ -418,18 +423,21 @@ module Fluent::Plugin
     def statistics
       stats = super
       services = service_discovery_services
-      healthy_nodes_count = 0
-      registed_nodes_count = services.size
+      @healthy_nodes_count_metrics.set(0)
+      @registered_nodes_count_metrics.set(services.size)
       services.each do |s|
         if s.available?
-          healthy_nodes_count += 1
+          @healthy_nodes_count_metrics.inc
         end
       end
 
-      stats.merge(
-        'healthy_nodes_count' => healthy_nodes_count,
-        'registered_nodes_count' => registed_nodes_count,
-      )
+      stats = {
+        'output' => stats["output"].merge({
+          'healthy_nodes_count' => @healthy_nodes_count_metrics.get,
+          'registered_nodes_count' => @registered_nodes_count_metrics.get,
+        })
+      }
+      stats
     end
 
     # MessagePack FixArray length is 3

data/lib/fluent/plugin/output.rb

@@ -14,6 +14,7 @@
 #    limitations under the License.
 #
 
+require 'fluent/env'
 require 'fluent/error'
 require 'fluent/plugin/base'
 require 'fluent/plugin/buffer'
@@ -37,7 +38,7 @@ module Fluent
       include PluginHelper::Mixin
       include UniqueId::Mixin
 
-      helpers_internal :thread, :retry_state
+      helpers_internal :thread, :retry_state, :metrics
 
       CHUNK_KEY_PATTERN = /^[-_.@a-zA-Z0-9]+$/
       CHUNK_KEY_PLACEHOLDER_PATTERN = /\$\{([-_.@$a-zA-Z0-9]+)\}/
@@ -164,7 +165,6 @@ module Fluent
       end
 
       attr_reader :as_secondary, :delayed_commit, :delayed_commit_timeout, :timekey_zone
-      attr_reader :num_errors, :emit_count, :emit_records, :write_count, :rollback_count
 
       # for tests
       attr_reader :buffer, :retry, :secondary, :chunk_keys, :chunk_key_accessors, :chunk_key_time, :chunk_key_tag
@@ -172,6 +172,30 @@ module Fluent
       # output_enqueue_thread_waiting: for test of output.rb itself
       attr_accessor :retry_for_error_chunk # if true, error flush will be retried even if under_plugin_development is true
 
+      def num_errors
+        @num_errors_metrics.get
+      end
+
+      def emit_count
+        @emit_count_metrics.get
+      end
+
+      def emit_size
+        @emit_size_metrics.get
+      end
+
+      def emit_records
+        @emit_records_metrics.get
+      end
+
+      def write_count
+        @write_count_metrics.get
+      end
+
+      def rollback_count
+        @rollback_count_metrics.get
+      end
+
       def initialize
         super
         @counter_mutex = Mutex.new
@@ -181,13 +205,15 @@ module Fluent
         @primary_instance = nil
 
         # TODO: well organized counters
-        @num_errors = 0
-        @emit_count = 0
-        @emit_records = 0
-        @write_count = 0
-        @rollback_count = 0
-        @flush_time_count = 0
-        @slow_flush_count = 0
+        @num_errors_metrics = nil
+        @emit_count_metrics = nil
+        @emit_records_metrics = nil
+        @emit_size_metrics = nil
+        @write_count_metrics = nil
+        @rollback_count_metrics = nil
+        @flush_time_count_metrics = nil
+        @slow_flush_count_metrics = nil
+        @enable_size_metrics = false
 
         # How to process events is decided here at once, but it will be decided in delayed way on #configure & #start
         if implement?(:synchronous)
@@ -246,6 +272,15 @@ module Fluent
 
         super
 
+        @num_errors_metrics = metrics_create(namespace: "fluentd", subsystem: "output", name: "num_errors", help_text: "Number of count num errors")
+        @emit_count_metrics = metrics_create(namespace: "fluentd", subsystem: "output", name: "emit_records", help_text: "Number of count emits")
+        @emit_records_metrics = metrics_create(namespace: "fluentd", subsystem: "output", name: "emit_records", help_text: "Number of emit records")
+        @emit_size_metrics =  metrics_create(namespace: "fluentd", subsystem: "output", name: "emit_size", help_text: "Total size of emit events")
+        @write_count_metrics = metrics_create(namespace: "fluentd", subsystem: "output", name: "write_count", help_text: "Number of writing events")
+        @rollback_count_metrics = metrics_create(namespace: "fluentd", subsystem: "output", name: "rollback_count", help_text: "Number of rollbacking operations")
+        @flush_time_count_metrics = metrics_create(namespace: "fluentd", subsystem: "output", name: "flush_time_count", help_text: "Count of flush time")
+        @slow_flush_count_metrics = metrics_create(namespace: "fluentd", subsystem: "output", name: "slow_flush_count", help_text: "Count of slow flush occurred time(s)")
+
         if has_buffer_section
           unless implement?(:buffered) || implement?(:delayed_commit)
             raise Fluent::ConfigError, "<buffer> section is configured, but plugin '#{self.class}' doesn't support buffering"
@@ -271,6 +306,8 @@ module Fluent
             @buffering = true
           end
         end
+        # Enable to update record size metrics or not
+        @enable_size_metrics = !!system_config.enable_size_metrics
 
         if @as_secondary
           if !@buffering && !@buffering.nil?
@@ -797,18 +834,19 @@ module Fluent
       end
 
       def emit_sync(tag, es)
-        @counter_mutex.synchronize{ @emit_count += 1 }
+        @emit_count_metrics.inc
         begin
           process(tag, es)
-          @counter_mutex.synchronize{ @emit_records += es.size }
+          @emit_records_metrics.add(es.size)
+          @emit_size_metrics.add(es.to_msgpack_stream.bytesize) if @enable_size_metrics
         rescue
-          @counter_mutex.synchronize{ @num_errors += 1 }
+          @num_errors_metrics.inc
           raise
         end
       end
 
       def emit_buffered(tag, es)
-        @counter_mutex.synchronize{ @emit_count += 1 }
+        @emit_count_metrics.inc
         begin
           execute_chunking(tag, es, enqueue: (@flush_mode == :immediate))
           if !@retry && @buffer.queued?(nil, optimistic: true)
@@ -816,7 +854,7 @@ module Fluent
           end
         rescue
           # TODO: separate number of errors into emit errors and write/flush errors
-          @counter_mutex.synchronize{ @num_errors += 1 }
+          @num_errors_metrics.inc
           raise
         end
       end
@@ -966,7 +1004,8 @@ module Fluent
         write_guard do
           @buffer.write(meta_and_data, enqueue: enqueue)
         end
-        @counter_mutex.synchronize{ @emit_records += records }
+        @emit_records_metrics.add(es.size)
+        @emit_size_metrics.add(es.to_msgpack_stream.bytesize) if @enable_size_metrics
         true
       end
 
@@ -983,7 +1022,8 @@ module Fluent
         write_guard do
           @buffer.write(meta_and_data, format: format_proc, enqueue: enqueue)
         end
-        @counter_mutex.synchronize{ @emit_records += records }
+        @emit_records_metrics.add(es.size)
+        @emit_size_metrics.add(es.to_msgpack_stream.bytesize) if @enable_size_metrics
         true
       end
 
@@ -1008,7 +1048,8 @@ module Fluent
         write_guard do
           @buffer.write({meta => data}, format: format_proc, enqueue: enqueue)
         end
-        @counter_mutex.synchronize{ @emit_records += records }
+        @emit_records_metrics.add(es.size)
+        @emit_size_metrics.add(es.to_msgpack_stream.bytesize) if @enable_size_metrics
         true
       end
 
@@ -1046,7 +1087,7 @@ module Fluent
         #         false if chunk was already flushed and couldn't be rollbacked unexpectedly
         # in many cases, false can be just ignored
         if @buffer.takeback_chunk(chunk_id)
-          @counter_mutex.synchronize{ @rollback_count += 1 }
+          @rollback_count_metrics.inc
           if update_retry
             primary = @as_secondary ? @primary_instance : self
             primary.update_retry_state(chunk_id, @as_secondary)
@@ -1062,7 +1103,7 @@ module Fluent
           while @dequeued_chunks.first && @dequeued_chunks.first.expired?
             info = @dequeued_chunks.shift
             if @buffer.takeback_chunk(info.chunk_id)
-              @counter_mutex.synchronize{ @rollback_count += 1 }
+              @rollback_count_metrics.inc
               log.warn "failed to flush the buffer chunk, timeout to commit.", chunk_id: dump_unique_id_hex(info.chunk_id), flushed_at: info.time
               primary = @as_secondary ? @primary_instance : self
               primary.update_retry_state(info.chunk_id, @as_secondary)
@@ -1077,7 +1118,7 @@ module Fluent
           until @dequeued_chunks.empty?
             info = @dequeued_chunks.shift
             if @buffer.takeback_chunk(info.chunk_id)
-              @counter_mutex.synchronize{ @rollback_count += 1 }
+              @rollback_count_metrics.inc
               log.info "delayed commit for buffer chunks was cancelled in shutdown", chunk_id: dump_unique_id_hex(info.chunk_id)
               primary = @as_secondary ? @primary_instance : self
               primary.update_retry_state(info.chunk_id, @as_secondary)
@@ -1120,7 +1161,7 @@ module Fluent
 
           if output.delayed_commit
             log.trace "executing delayed write and commit", chunk: dump_unique_id_hex(chunk.unique_id)
-            @counter_mutex.synchronize{ @write_count += 1 }
+            @write_count_metrics.inc
             @dequeued_chunks_mutex.synchronize do
               # delayed_commit_timeout for secondary is configured in <buffer> of primary (<secondary> don't get <buffer>)
               @dequeued_chunks << DequeuedChunkInfo.new(chunk.unique_id, Time.now, self.delayed_commit_timeout)
@@ -1132,7 +1173,7 @@ module Fluent
             chunk_id = chunk.unique_id
             dump_chunk_id = dump_unique_id_hex(chunk_id)
             log.trace "adding write count", instance: self.object_id
-            @counter_mutex.synchronize{ @write_count += 1 }
+            @write_count_metrics.inc
             log.trace "executing sync write", chunk: dump_chunk_id
 
             output.write(chunk)
@@ -1188,7 +1229,7 @@ module Fluent
           end
 
           if @buffer.takeback_chunk(chunk.unique_id)
-            @counter_mutex.synchronize { @rollback_count += 1 }
+            @rollback_count_metrics.inc
           end
 
           update_retry_state(chunk.unique_id, using_secondary, e)
@@ -1208,8 +1249,8 @@ module Fluent
           backup_dir = File.dirname(backup_file)
 
           log.warn "bad chunk is moved to #{backup_file}"
-          FileUtils.mkdir_p(backup_dir, mode: system_config.dir_permission || 0755) unless Dir.exist?(backup_dir)
-          File.open(backup_file, 'ab', system_config.file_permission || 0644) { |f|
+          FileUtils.mkdir_p(backup_dir, mode: system_config.dir_permission || Fluent::DEFAULT_DIR_PERMISSION) unless Dir.exist?(backup_dir)
+          File.open(backup_file, 'ab', system_config.file_permission || Fluent::DEFAULT_FILE_PERMISSION) { |f|
             chunk.write_to(f)
           }
         end
@@ -1219,9 +1260,9 @@ module Fluent
       def check_slow_flush(start)
         elapsed_time = Fluent::Clock.now - start
         elapsed_millsec = (elapsed_time * 1000).to_i
-        @counter_mutex.synchronize { @flush_time_count += elapsed_millsec }
+        @flush_time_count_metrics.add(elapsed_millsec)
         if elapsed_time > @slow_flush_log_threshold
-          @counter_mutex.synchronize { @slow_flush_count += 1 }
+          @slow_flush_count_metrics.inc
           log.warn "buffer flush took longer time than slow_flush_log_threshold:",
                    elapsed_time: elapsed_time, slow_flush_log_threshold: @slow_flush_log_threshold, plugin_id: self.plugin_id
         end
@@ -1229,13 +1270,13 @@ module Fluent
 
       def update_retry_state(chunk_id, using_secondary, error = nil)
         @retry_mutex.synchronize do
-          @counter_mutex.synchronize{ @num_errors += 1 }
+          @num_errors_metrics.inc
           chunk_id_hex = dump_unique_id_hex(chunk_id)
 
           unless @retry
             @retry = retry_state(@buffer_config.retry_randomize)
             if error
-              log.warn "failed to flush the buffer.", retry_time: @retry.steps, next_retry_seconds: @retry.next_time, chunk: chunk_id_hex, error: error
+              log.warn "failed to flush the buffer.", retry_times: @retry.steps, next_retry_time: @retry.next_time.round, chunk: chunk_id_hex, error: error
               log.warn_backtrace error.backtrace
             end
             return
@@ -1264,11 +1305,11 @@ module Fluent
             if error
               if using_secondary
                 msg = "failed to flush the buffer with secondary output."
-                log.warn msg, retry_time: @retry.steps, next_retry_seconds: @retry.next_time, chunk: chunk_id_hex, error: error
+                log.warn msg, retry_times: @retry.steps, next_retry_time: @retry.next_time.round, chunk: chunk_id_hex, error: error
                 log.warn_backtrace error.backtrace
               else
                 msg = "failed to flush the buffer."
-                log.warn msg, retry_time: @retry.steps, next_retry_seconds: @retry.next_time, chunk: chunk_id_hex, error: error
+                log.warn msg, retry_times: @retry.steps, next_retry_time: @retry.next_time.round, chunk: chunk_id_hex, error: error
                 log.warn_backtrace error.backtrace
               end
             end
@@ -1490,15 +1531,16 @@ module Fluent
 
       def statistics
         stats = {
-          'emit_records' => @emit_records,
+          'emit_records' => @emit_records_metrics.get,
+          'emit_size' => @emit_size_metrics.get,
           # Respect original name
           # https://github.com/fluent/fluentd/blob/45c7b75ba77763eaf87136864d4942c4e0c5bfcd/lib/fluent/plugin/in_monitor_agent.rb#L284
-          'retry_count' => @num_errors,
-          'emit_count' => @emit_count,
-          'write_count' => @write_count,
-          'rollback_count' => @rollback_count,
-          'slow_flush_count' => @slow_flush_count,
-          'flush_time_count' => @flush_time_count,
+          'retry_count' => @num_errors_metrics.get,
+          'emit_count' => @emit_count_metrics.get,
+          'write_count' => @write_count_metrics.get,
+          'rollback_count' => @rollback_count_metrics.get,
+          'slow_flush_count' => @slow_flush_count_metrics.get,
+          'flush_time_count' => @flush_time_count_metrics.get,
         }
 
         if @buffer && @buffer.respond_to?(:statistics)

data/lib/fluent/plugin/parser_apache2.rb

@@ -21,7 +21,7 @@ module Fluent
     class Apache2Parser < Parser
       Plugin.register_parser('apache2', self)
 
-      REGEXP = /^(?<host>[^ ]*) [^ ]* (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>(?:[^\"]|\\.)*?)(?: +\S*)?)?" (?<code>[^ ]*) (?<size>[^ ]*)(?: "(?<referer>(?:[^\"]|\\.)*)" "(?<agent>(?:[^\"]|\\.)*)")?$/
+      REGEXP = /^(?<host>[^ ]*) [^ ]* (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>(?:[^\"]|\\")*?)(?: +\S*)?)?" (?<code>[^ ]*) (?<size>[^ ]*)(?: "(?<referer>(?:[^\"]|\\")*)" "(?<agent>(?:[^\"]|\\")*)")?$/
       TIME_FORMAT = "%d/%b/%Y:%H:%M:%S %z"
 
       def initialize

data/lib/fluent/plugin/storage_local.rb

@@ -14,6 +14,7 @@
 #    limitations under the License.
 #
 
+require 'fluent/env'
 require 'fluent/plugin'
 require 'fluent/plugin/storage'
 
@@ -25,14 +26,11 @@ module Fluent
     class LocalStorage < Storage
       Fluent::Plugin.register_storage('local', self)
 
-      DEFAULT_DIR_MODE = 0755
-      DEFAULT_FILE_MODE = 0644
-
       config_param :path, :string, default: nil
-      config_param :mode, default: DEFAULT_FILE_MODE do |v|
+      config_param :mode, default: Fluent::DEFAULT_FILE_PERMISSION do |v|
         v.to_i(8)
       end
-      config_param :dir_mode, default: DEFAULT_DIR_MODE do |v|
+      config_param :dir_mode, default: Fluent::DEFAULT_DIR_PERMISSION do |v|
         v.to_i(8)
       end
       config_param :pretty_print, :bool, default: false