fluentd 1.2.3 → 1.2.4.rc1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 592df476adfeb30ef3a08865d03af3eeb7f60e1d27684f22dc3505b8dc7bcab4
-  data.tar.gz: acc575bbf086686addf560a7a65e084221a609ce5eff6385b88a9aea65c4aca4
+  metadata.gz: 14bc521f8482997b686c3c465c84dd7a3ef10ac52e208883c068394384163f66
+  data.tar.gz: aefedb8e2a5917a6bc37c56ed4d9c33e7ba76a0883d4a877e932cc681a610d82
 SHA512:
-  metadata.gz: c1f5ebc8c426c80c7b6501afba2ac9534aeceafe21945e58fd0530dbde099d8cdbd0a6c9533c7500d838db09f66dfbefd4fbed5cb6304ab71fe473ab0a63cd5c
-  data.tar.gz: 6eef15e5140fc4547c59215ae01cd9eb661429bb7eb36067e6834f877c916bf78507345091002116129435e548525654e289db36c3a786f7aebc5780835458ef
+  metadata.gz: e767a398beb8858af702625ae9d900aac20eea0b8cf46efd126cd9281b1acb0a93bc66d5485fef0276d0ecf7e573ec092cad96d6f9146960a9ac6d5c79215604
+  data.tar.gz: dea6a30c9827ece7cb88801eebff7bd3425807e5d129400284d36f0059add10beb54390890e76c8b9de3eb9ec82aeab32322fd4aa41e36bb3487c333bee41a64

data/lib/fluent/plugin/output.rb

@@ -304,6 +304,10 @@ module Fluent
             raise Fluent::ConfigError, "<buffer ...> argument includes 'time', but timekey is not configured" unless @buffer_config.timekey
             Fluent::Timezone.validate!(@buffer_config.timekey_zone)
             @timekey_zone = @buffer_config.timekey_use_utc ? '+0000' : @buffer_config.timekey_zone
+            @timekey = @buffer_config.timekey
+            @timekey_use_utc = @buffer_config.timekey_use_utc
+            @offset = Fluent::Timezone.utc_offset(@timekey_zone)
+            @calculate_offset = @offset.respond_to?(:call) ? @offset : nil
             @output_time_formatter_cache = {}
           end
 
@@ -803,20 +807,17 @@ module Fluent
           if !@chunk_key_time && !@chunk_key_tag
             @buffer.metadata()
           elsif @chunk_key_time && @chunk_key_tag
-            time_int = time.to_i
-            timekey = (time_int - (time_int % @buffer_config.timekey)).to_i
+            timekey = calculate_timekey(time)
             @buffer.metadata(timekey: timekey, tag: tag)
           elsif @chunk_key_time
-            time_int = time.to_i
-            timekey = (time_int - (time_int % @buffer_config.timekey)).to_i
+            timekey = calculate_timekey(time)
             @buffer.metadata(timekey: timekey)
           else
             @buffer.metadata(tag: tag)
           end
         else
           timekey = if @chunk_key_time
-                      time_int = time.to_i
-                      (time_int - (time_int % @buffer_config.timekey)).to_i
+                      calculate_timekey(time)
                     else
                       nil
                     end
@@ -825,6 +826,16 @@ module Fluent
         end
       end
 
+      def calculate_timekey(time)
+        time_int = time.to_i
+        if @timekey_use_utc
+          (time_int - (time_int % @timekey)).to_i
+        else
+          offset = @calculate_offset ? @calculate_offset.call(time) : @offset
+          (time_int - ((time_int + offset)% @timekey)).to_i
+        end
+      end
+
       def chunk_for_test(tag, time, record)
         require 'fluent/plugin/buffer/memory_chunk'
 

data/lib/fluent/root_agent.rb

@@ -318,7 +318,7 @@ module Fluent
         log.warn "send an error event stream to @ERROR:", error_info
         @error_collector.emit_stream(tag, es)
       else
-        now = Time.now
+        now = Time.now.to_i
         if @suppress_emit_error_log_interval.zero? || now > @next_emit_error_log_time
           log.warn "emit transaction failed:", error_info
           log.warn_backtrace
@@ -347,7 +347,7 @@ module Fluent
       end
 
       def handle_emits_error(tag, es, e)
-        now = EventTime.now
+        now = EventTime.now.to_i
         if @suppress_emit_error_log_interval.zero? || now > @next_emit_error_log_time
           log.warn "emit transaction failed in @ERROR:", error: e, tag: tag
           log.warn_backtrace

data/lib/fluent/timezone.rb

@@ -139,5 +139,19 @@ module Fluent
 
       return nil
     end
+
+    def self.utc_offset(timezone)
+      return 0 if timezone.nil?
+
+      case timezone
+      when NUMERIC_PATTERN
+        Time.zone_offset(timezone)
+      when NAME_PATTERN
+        tz = TZInfo::Timezone.get(timezone)
+        ->(time) {
+          tz.period_for_utc(time).utc_total_offset
+        }
+      end
+    end
   end
 end

data/lib/fluent/version.rb

@@ -16,6 +16,6 @@
 
 module Fluent
 
-  VERSION = '1.2.3'
+  VERSION = '1.2.4.rc1'
 
 end

data/test/plugin/test_out_file.rb

@@ -561,6 +561,79 @@ class FileOutputTest < Test::Unit::TestCase
     check_gzipped_result(path, formatted_lines * 3)
   end
 
+  test 'append when JST' do
+    with_timezone(Fluent.windows? ? "JST-9" : "Asia/Tokyo") do
+      time = event_time("2011-01-02 03:14:15+09:00")
+      formatted_lines = %[2011-01-02T03:14:15+09:00\ttest\t{"a":1}\n] + %[2011-01-02T03:14:15+09:00\ttest\t{"a":2}\n]
+
+      write_once = ->(){
+        d = create_driver %[
+          path #{TMP_DIR}/out_file_test
+          compress gz
+          append true
+          <buffer>
+            timekey_use_utc false
+            timekey_zone Asia/Tokyo
+          </buffer>
+        ]
+        d.run(default_tag: 'test'){
+          d.feed(time, {"a"=>1})
+          d.feed(time, {"a"=>2})
+        }
+        d.instance.last_written_path
+      }
+
+      path = write_once.call
+      assert_equal "#{TMP_DIR}/out_file_test.20110102.log.gz", path
+      check_gzipped_result(path, formatted_lines)
+
+      path = write_once.call
+      assert_equal "#{TMP_DIR}/out_file_test.20110102.log.gz", path
+      check_gzipped_result(path, formatted_lines * 2)
+
+      path = write_once.call
+      assert_equal "#{TMP_DIR}/out_file_test.20110102.log.gz", path
+      check_gzipped_result(path, formatted_lines * 3)
+    end
+  end
+
+  test 'append when UTC-02 but timekey_zone is +0900' do
+    with_timezone("UTC-02") do # +0200
+      time = event_time("2011-01-02 17:14:15+02:00")
+      formatted_lines = %[2011-01-02T17:14:15+02:00\ttest\t{"a":1}\n] + %[2011-01-02T17:14:15+02:00\ttest\t{"a":2}\n]
+
+      write_once = ->(){
+        d = create_driver %[
+          path #{TMP_DIR}/out_file_test
+          compress gz
+          append true
+          <buffer>
+            timekey_use_utc false
+            timekey_zone +0900
+          </buffer>
+        ]
+        d.run(default_tag: 'test'){
+          d.feed(time, {"a"=>1})
+          d.feed(time, {"a"=>2})
+        }
+        d.instance.last_written_path
+      }
+
+      path = write_once.call
+      # Rotated at 2011-01-02 17:00:00+02:00
+      assert_equal "#{TMP_DIR}/out_file_test.20110103.log.gz", path
+      check_gzipped_result(path, formatted_lines)
+
+      path = write_once.call
+      assert_equal "#{TMP_DIR}/out_file_test.20110103.log.gz", path
+      check_gzipped_result(path, formatted_lines * 2)
+
+      path = write_once.call
+      assert_equal "#{TMP_DIR}/out_file_test.20110103.log.gz", path
+      check_gzipped_result(path, formatted_lines * 3)
+    end
+  end
+
   test '${chunk_id}' do
     time = event_time("2011-01-02 13:14:15 UTC")
     formatted_lines = %[2011-01-02T13:14:15Z\ttest\t{"a":1}\n] + %[2011-01-02T13:14:15Z\ttest\t{"a":2}\n]

data/test/test_root_agent.rb

@@ -1,6 +1,7 @@
 require_relative 'helper'
 require 'fluent/event_router'
 require 'fluent/system_config'
+require 'timecop'
 require_relative 'test_plugin_classes'
 
 class RootAgentTest < ::Test::Unit::TestCase
@@ -609,6 +610,38 @@ EOC
     end
   end
 
+  sub_test_case 'configure emit_error_interval' do
+    setup do
+      system_config = SystemConfig.new
+      system_config.emit_error_log_interval = 30
+      @ra = RootAgent.new(log: $log, system_config: system_config)
+      stub(Engine).root_agent { @ra }
+      @ra.log.out.reset
+      one_minute_ago = Time.now.to_i - 60
+      Timecop.freeze(one_minute_ago)
+    end
+
+    teardown do
+      Timecop.return
+    end
+
+    test 'suppresses errors' do
+      mock(@ra.log).warn_backtrace()
+      e = StandardError.new('standard error')
+      begin
+        @ra.handle_emits_error("tag", nil, e)
+      rescue
+      end
+
+      begin
+      @ra.handle_emits_error("tag", nil, e)
+      rescue
+      end
+
+      assert_equal 1, @ra.log.out.logs.size
+    end
+  end
+
   sub_test_case 'configured at worker2 with 4 workers environment' do
     setup do
       ENV['SERVERENGINE_WORKER_ID'] = '2'

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluentd
 version: !ruby/object:Gem::Version
-  version: 1.2.3
+  version: 1.2.4.rc1
 platform: ruby
 authors:
 - Sadayuki Furuhashi
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-07-11 00:00:00.000000000 Z
+date: 2018-07-23 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: msgpack
@@ -764,9 +764,9 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '2.1'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">="
+  - - ">"
     - !ruby/object:Gem::Version
-      version: '0'
+      version: 1.3.1
 requirements: []
 rubyforge_project: 
 rubygems_version: 2.7.6