fluent-plugin-google-cloud 0.7.28 → 0.7.29

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 6557bd45ff7ce7fffe52375ece31a52e871c139465f799701fc2ce493cf1e0a7
-  data.tar.gz: 25441416896fd09d2e3e8fd2ed95228772d2ee10a6ba5976c07b6499e0df67b5
+  metadata.gz: e47e2821d1838929cf6b2181462d8c2aed2d4a607b2ad4cd0711a7c2d25a5edd
+  data.tar.gz: 0c2d369dfad54166a1ca2f2810f72e69a252212e9b243a7bea97b2afa231481e
 SHA512:
-  metadata.gz: ec86b91be90466150a1f7e2bdc21bdf64481586a7e860bf7dafdc1a5e0b1e7c6be085b2268ac33dfffc966ecd5e301ee073369086594493f1b4d121821ed2a2c
-  data.tar.gz: ea9d2c7a0afbba9450f41759b1d371a39d63b1660a0e2a9d1186dded1d9a885fb00693dcb341ff705d26a0d09086c654c590ab4a3db33e61f57358dd4a30c632
+  metadata.gz: d528607184bfc1beb617f187919c2b390344137cc5c01c0506fede8bef2cd237faf645354d7a957962ffe8263d2c496d712544ce79d6a6c238b91de4942f3ddf
+  data.tar.gz: 96ce50ffcd3df259007bb3b73c1d7301d5ac95f93c4862627125dfcd6d20d43c8b75fa2b07578ad324fc430741f36d76457fd4098d01ec4f3d15437928c04c11

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    fluent-plugin-google-cloud (0.7.28)
+    fluent-plugin-google-cloud (0.7.29)
       fluentd (= 1.6.3)
       google-api-client (= 0.30.8)
       google-cloud-logging (= 1.6.6)
@@ -52,10 +52,12 @@ GEM
       representable (~> 3.0)
       retriable (>= 2.0, < 4.0)
       signet (~> 0.10)
-    google-cloud-core (1.4.1)
+    google-cloud-core (1.5.0)
       google-cloud-env (~> 1.0)
+      google-cloud-errors (~> 1.0)
     google-cloud-env (1.3.0)
       faraday (~> 0.11)
+    google-cloud-errors (1.0.0)
     google-cloud-logging (1.6.6)
       concurrent-ruby (~> 1.1)
       google-cloud-core (~> 1.2)
@@ -113,7 +115,7 @@ GEM
       google-cloud-trace (~> 0.33)
       opencensus (~> 0.5)
     os (1.0.1)
-    parser (2.7.0.1)
+    parser (2.7.0.2)
       ast (~> 2.4.0)
     power_assert (1.1.5)
     powerpack (0.1.2)
@@ -161,14 +163,14 @@ GEM
       power_assert
     thor (1.0.1)
     thread_safe (0.3.6)
-    tins (1.23.0)
+    tins (1.24.0)
       sync
     tzinfo (1.2.6)
       thread_safe (~> 0.1)
     tzinfo-data (1.2019.3)
       tzinfo (>= 1.0.0)
     uber (0.1.0)
-    unicode-display_width (1.6.0)
+    unicode-display_width (1.6.1)
     webmock (3.6.2)
       addressable (>= 2.3.6)
       crack (>= 0.3.2)

data/fluent-plugin-google-cloud.gemspec

@@ -10,7 +10,7 @@ eos
   gem.homepage      =
     'https://github.com/GoogleCloudPlatform/fluent-plugin-google-cloud'
   gem.license       = 'Apache-2.0'
-  gem.version       = '0.7.28'
+  gem.version       = '0.7.29'
   gem.authors       = ['Stackdriver Agents Team']
   gem.email         = ['stackdriver-agents@google.com']
   gem.required_ruby_version = Gem::Requirement.new('>= 2.2')

data/lib/fluent/plugin/out_google_cloud.rb

@@ -724,7 +724,10 @@ module Fluent
             end
           end
 
-          ts_secs, ts_nanos = compute_timestamp(record, time)
+          ts_secs, ts_nanos, timestamp = compute_timestamp(record, time)
+          ts_secs, ts_nanos = adjust_timestamp_if_invalid(timestamp, Time.now) \
+            if @adjust_invalid_timestamps && timestamp
+
           severity = compute_severity(
             entry_level_resource.type, record, entry_level_common_labels)
 
@@ -1663,7 +1666,6 @@ module Fluent
     end
 
     def compute_timestamp(record, time)
-      current_time = Time.now
       if record.key?('timestamp') &&
          record['timestamp'].is_a?(Hash) &&
          record['timestamp'].key?('seconds') &&
@@ -1715,32 +1717,43 @@ module Fluent
                    ts_nanos
                  end
 
-      if @adjust_invalid_timestamps && timestamp
-        # Adjust timestamps from the future.
-        # There are two cases:
-        # 1. The parsed timestamp is later in the current year:
-        # This can happen when system log lines from previous years are missing
-        # the year, so the date parser assumes the current year.
-        # We treat these lines as coming from last year. This could label
-        # 2-year-old logs incorrectly, but this probably isn't super important.
-        #
-        # 2. The parsed timestamp is past the end of the current year:
-        # Since the year is different from the current year, this isn't the
-        # missing year in system logs. It is unlikely that users explicitly
-        # write logs at a future date. This could result from an unsynchronized
-        # clock on a VM, or some random value being parsed as the timestamp.
-        # We reset the timestamp on those lines to the default value and let the
-        # downstream API handle it.
-        next_year = Time.mktime(current_time.year + 1)
-        one_day_later = current_time.to_datetime.next_day.to_time
-        if timestamp >= next_year # Case 2.
-          ts_secs = 0
-          ts_nanos = 0
-        elsif timestamp >= one_day_later # Case 1.
-          adjusted_timestamp = timestamp.to_datetime.prev_year.to_time
-          ts_secs = adjusted_timestamp.tv_sec
-          # The value of ts_nanos should not change when subtracting a year.
-        end
+      [ts_secs, ts_nanos, timestamp]
+    end
+
+    # Adjust timestamps from the future.
+    # The base case is:
+    # 0. The parsed timestamp is less than one day into the future.
+    # This is allowed by the API, and should be left unchanged.
+    #
+    # Beyond that, there are two cases:
+    # 1. The parsed timestamp is later in the current year:
+    # This can happen when system log lines from previous years are missing
+    # the year, so the date parser assumes the current year.
+    # We treat these lines as coming from last year. This could label
+    # 2-year-old logs incorrectly, but this probably isn't super important.
+    #
+    # 2. The parsed timestamp is past the end of the current year:
+    # Since the year is different from the current year, this isn't the
+    # missing year in system logs. It is unlikely that users explicitly
+    # write logs at a future date. This could result from an unsynchronized
+    # clock on a VM, or some random value being parsed as the timestamp.
+    # We reset the timestamp on those lines to the default value and let the
+    # downstream API handle it.
+    def adjust_timestamp_if_invalid(timestamp, current_time)
+      ts_secs = timestamp.tv_sec
+      ts_nanos = timestamp.tv_nsec
+
+      next_year = Time.mktime(current_time.year + 1)
+      one_day_later = current_time.to_datetime.next_day.to_time
+      if timestamp < one_day_later # Case 0.
+        # Leave the timestamp as-is.
+      elsif timestamp >= next_year # Case 2.
+        ts_secs = 0
+        ts_nanos = 0
+      else # Case 1.
+        adjusted_timestamp = timestamp.to_datetime.prev_year.to_time
+        ts_secs = adjusted_timestamp.tv_sec
+        # The value of ts_nanos should not change when subtracting a year.
       end
 
       [ts_secs, ts_nanos]

data/test/plugin/base_test.rb

@@ -943,77 +943,206 @@ module BaseTest
     end
   end
 
-  def test_timestamps
+  def test_compute_timestamp
     setup_gce_metadata_stubs
-    current_time = Time.now
+    d = create_driver(APPLICATION_DEFAULT_CONFIG)
+
+    compute_timestamp = lambda do |driver, record, time|
+      driver.instance.send(:compute_timestamp, record, time)
+    end
+
+    current_time = Time.new(2019, 12, 29, 10, 23, 35, '-08:00')
+    one_day_later = current_time.to_datetime.next_day.to_time
+    just_under_one_day_later = one_day_later - 1
     next_year = Time.mktime(current_time.year + 1)
     one_second_before_next_year = next_year - 1
-    adjusted_to_last_year =
-      one_second_before_next_year.to_datetime.prev_year.to_time
     one_second_into_next_year = next_year + 1
-    one_day_into_next_year = next_year.to_date.next_day.to_time
+    one_day_into_next_year = next_year.to_datetime.next_day.to_time
+
+    [
+      Time.at(123_456.789),
+      Time.at(0),
+      current_time,
+      just_under_one_day_later,
+      one_second_before_next_year,
+      next_year,
+      one_second_into_next_year,
+      one_day_into_next_year
+    ].each do |ts|
+      # Use record collection time.
+      ts_secs, ts_nanos, actual_ts = compute_timestamp[d, {
+        'message' => ''
+      }, ts.to_f]
+      assert_timestamp_matches ts, ts_secs, ts_nanos, actual_ts.iso8601
+
+      # Use the (deprecated) timeNanos key.
+      ts_secs, ts_nanos, actual_ts = compute_timestamp[d, {
+        'message' => '',
+        'timeNanos' => ts.tv_sec * 1_000_000_000 + ts.tv_nsec
+      }, 1.0]
+      assert_timestamp_matches ts, ts_secs, ts_nanos, actual_ts.iso8601
+
+      # Use the structured timestamp key.
+      ts_secs, ts_nanos, actual_ts = compute_timestamp[d, {
+        'message' => '',
+        'timestamp' => { 'seconds' => ts.tv_sec, 'nanos' => ts.tv_nsec }
+      }, 1.0]
+      assert_timestamp_matches ts, ts_secs, ts_nanos, actual_ts.iso8601
+
+      # Use the timestampSeconds/timestampNanos keys.
+      ts_secs, ts_nanos, actual_ts = compute_timestamp[d, {
+        'message' => '',
+        'timestampSeconds' => ts.tv_sec,
+        'timestampNanos' => ts.tv_nsec
+      }, 1.0]
+      assert_timestamp_matches ts, ts_secs, ts_nanos, actual_ts.iso8601
+
+      # Use the string timestampSeconds/timestampNanos keys.
+      ts_secs, ts_nanos, actual_ts = compute_timestamp[d, {
+        'message' => '',
+        'timestampSeconds' => ts.tv_sec.to_s,
+        'timestampNanos' => ts.tv_nsec.to_s
+      }, 1.0]
+      assert_timestamp_matches ts, ts_secs, ts_nanos, actual_ts.iso8601
+    end
+  end
+
+  def test_adjust_timestamp
+    setup_gce_metadata_stubs
+    d = create_driver(APPLICATION_DEFAULT_CONFIG)
+
+    adjust_timestamp_if_invalid = lambda do |driver, timestamp, current_time|
+      driver.instance.send(:adjust_timestamp_if_invalid, timestamp,
+                           current_time)
+    end
+
+    december_29 = Time.new(2019, 12, 29, 10, 23, 35, '-08:00')
+    december_31 = Time.new(2019, 12, 31, 10, 23, 35, '-08:00')
+    january_1 = Time.new(2020, 1, 1, 10, 23, 35, '-08:00')
+
     {
-      APPLICATION_DEFAULT_CONFIG => {
-        Time.at(123_456.789) => Time.at(123_456.789),
-        Time.at(0) => Time.at(0),
-        current_time => current_time,
-        one_second_before_next_year => adjusted_to_last_year,
-        next_year => Time.at(0),
-        one_second_into_next_year => Time.at(0),
-        one_day_into_next_year => Time.at(0)
-      },
-      NO_ADJUST_TIMESTAMPS_CONFIG => {
-        Time.at(123_456.789) => Time.at(123_456.789),
-        Time.at(0) => Time.at(0),
-        current_time => current_time,
-        one_second_before_next_year => one_second_before_next_year,
-        next_year => next_year,
-        one_second_into_next_year => one_second_into_next_year,
-        one_day_into_next_year => one_day_into_next_year
-      }
-    }.each do |config, timestamps|
+      # December 29, 2019 (normal operation).
+      december_29 => begin
+        one_day_later = Time.new(2019, 12, 30, 10, 23, 35, '-08:00')
+        one_day_a_year_earlier = Time.new(2018, 12, 30, 10, 23, 35, '-08:00')
+        just_under_one_day_later = Time.new(2019, 12, 30, 10, 23, 34, '-08:00')
+        next_year = Time.new(2020, 1, 1, 0, 0, 0, '-08:00')
+        one_second_before_next_year =
+          Time.new(2019, 12, 31, 11, 59, 59, '-08:00')
+        one_second_before_this_year =
+          Time.new(2018, 12, 31, 11, 59, 59, '-08:00')
+        one_second_into_next_year = Time.new(2020, 1, 1, 0, 0, 1, '-08:00')
+        one_day_into_next_year = Time.new(2020, 1, 2, 0, 0, 0, '-08:00')
+        {
+          Time.at(123_456.789) => Time.at(123_456.789),
+          Time.at(0) => Time.at(0),
+          december_29 => december_29,
+          one_day_later => one_day_a_year_earlier,
+          just_under_one_day_later => just_under_one_day_later,
+          one_second_before_next_year => one_second_before_this_year,
+          next_year => Time.at(0),
+          one_second_into_next_year => Time.at(0),
+          one_day_into_next_year => Time.at(0)
+        }
+      end,
+      # January 1, 2020 (normal operation).
+      january_1 => begin
+        one_day_later = Time.new(2020, 1, 2, 10, 23, 35, '-08:00')
+        one_day_a_year_earlier = Time.new(2019, 1, 2, 10, 23, 35, '-08:00')
+        just_under_one_day_later = Time.new(2020, 1, 2, 10, 23, 34, '-08:00')
+        next_year = Time.new(2021, 1, 1, 0, 0, 0, '-08:00')
+        one_second_before_next_year =
+          Time.new(2020, 12, 31, 11, 59, 59, '-08:00')
+        one_second_before_this_year =
+          Time.new(2019, 12, 31, 11, 59, 59, '-08:00')
+        one_second_into_next_year = Time.new(2021, 1, 1, 0, 0, 1, '-08:00')
+        one_day_into_next_year = Time.new(2021, 1, 2, 0, 0, 0, '-08:00')
+        {
+          Time.at(123_456.789) => Time.at(123_456.789),
+          Time.at(0) => Time.at(0),
+          january_1 => january_1,
+          one_day_later => one_day_a_year_earlier,
+          just_under_one_day_later => just_under_one_day_later,
+          one_second_before_next_year => one_second_before_this_year,
+          next_year => Time.at(0),
+          one_second_into_next_year => Time.at(0),
+          one_day_into_next_year => Time.at(0)
+        }
+      end,
+      # December 31, 2019 (next day overlaps new year).
+      december_31 => begin
+        one_day_later = Time.new(2020, 1, 1, 10, 23, 35, '-08:00')
+        just_under_one_day_later = Time.new(2020, 1, 1, 10, 23, 34, '-08:00')
+        next_year = Time.new(2020, 1, 1, 0, 0, 0, '-08:00')
+        one_second_before_next_year =
+          Time.new(2019, 12, 31, 11, 59, 59, '-08:00')
+        one_second_into_next_year = Time.new(2020, 1, 1, 0, 0, 1, '-08:00')
+        one_day_into_next_year = Time.new(2020, 1, 2, 0, 0, 0, '-08:00')
+        {
+          Time.at(123_456.789) => Time.at(123_456.789),
+          Time.at(0) => Time.at(0),
+          december_31 => december_31,
+          one_day_later => Time.at(0), # Falls into the next year.
+          just_under_one_day_later => just_under_one_day_later,
+          one_second_before_next_year => one_second_before_next_year,
+          next_year => next_year,
+          one_second_into_next_year => one_second_into_next_year,
+          one_day_into_next_year => Time.at(0)
+        }
+      end
+    }.each do |current_time, timestamps|
       timestamps.each do |ts, expected_ts|
-        emit_index = 0
-        setup_logging_stubs do
-          @logs_sent = []
-          d = create_driver(config)
-          # Test the "native" fluentd timestamp as well as our nanosecond tags.
-          d.emit({ 'message' => log_entry(emit_index) }, ts.to_f)
-          emit_index += 1
-          d.emit('message' => log_entry(emit_index),
-                 'timeNanos' => ts.tv_sec * 1_000_000_000 + ts.tv_nsec)
-          emit_index += 1
-          d.emit('message' => log_entry(emit_index),
-                 'timestamp' => { 'seconds' => ts.tv_sec,
-                                  'nanos' => ts.tv_nsec })
-          emit_index += 1
-          d.emit('message' => log_entry(emit_index),
-                 'timestampSeconds' => ts.tv_sec,
-                 'timestampNanos' => ts.tv_nsec)
-          emit_index += 1
-          d.emit('message' => log_entry(emit_index),
-                 'timestampSeconds' => ts.tv_sec.to_s,
-                 'timestampNanos' => ts.tv_nsec.to_s)
-          emit_index += 1
-          d.run
-          verify_log_entries(emit_index, COMPUTE_PARAMS) do |entry, i|
-            verify_default_log_entry_text(entry['textPayload'], i, entry)
-            actual_timestamp = timestamp_parse(entry['timestamp'])
-            assert_equal actual_timestamp['seconds'], expected_ts.tv_sec, entry
-            # Fluentd v0.14 onwards supports nanosecond timestamp values.
-            # Added in 600 ns delta to avoid flaky tests introduced
-            # due to rounding error in double-precision floating-point numbers
-            # (to account for the missing 9 bits of precision ~ 512 ns).
-            # See http://wikipedia.org/wiki/Double-precision_floating-point_format.
-            assert_in_delta expected_ts.tv_nsec, actual_timestamp['nanos'],
-                            600, entry
-          end
+        ts_secs, ts_nanos = adjust_timestamp_if_invalid[d, ts, current_time]
+        adjusted_ts = Time.at(ts_secs, ts_nanos / 1_000.0)
+        assert_timestamp_matches expected_ts, ts_secs, ts_nanos,
+                                 adjusted_ts.iso8601
+      end
+    end
+  end
+
+  def test_log_timestamps
+    setup_gce_metadata_stubs
+    current_time = Time.now
+    {
+      # Verify that timestamps make it through.
+      Time.at(123_456.789) => Time.at(123_456.789),
+      Time.at(0) => Time.at(0),
+      current_time => current_time
+    }.each do |ts, expected_ts|
+      emit_index = 0
+      setup_logging_stubs do
+        @logs_sent = []
+        d = create_driver(APPLICATION_DEFAULT_CONFIG)
+        # Test the "native" fluentd timestamp as well as our nanosecond tags.
+        d.emit({ 'message' => log_entry(emit_index) }, ts.to_f)
+        emit_index += 1
+        d.emit('message' => log_entry(emit_index),
+               'timeNanos' => ts.tv_sec * 1_000_000_000 + ts.tv_nsec)
+        emit_index += 1
+        d.emit('message' => log_entry(emit_index),
+               'timestamp' => { 'seconds' => ts.tv_sec,
+                                'nanos' => ts.tv_nsec })
+        emit_index += 1
+        d.emit('message' => log_entry(emit_index),
+               'timestampSeconds' => ts.tv_sec,
+               'timestampNanos' => ts.tv_nsec)
+        emit_index += 1
+        d.emit('message' => log_entry(emit_index),
+               'timestampSeconds' => ts.tv_sec.to_s,
+               'timestampNanos' => ts.tv_nsec.to_s)
+        emit_index += 1
+        d.run
+        verify_log_entries(emit_index, COMPUTE_PARAMS) do |entry, i|
+          verify_default_log_entry_text(entry['textPayload'], i, entry)
+          actual_timestamp = timestamp_parse(entry['timestamp'])
+          assert_timestamp_matches expected_ts, actual_timestamp['seconds'],
+                                   actual_timestamp['nanos'], entry
         end
       end
     end
   end
 
-  def test_malformed_timestamp
+  def test_malformed_timestamp_field
     setup_gce_metadata_stubs
     setup_logging_stubs do
       d = create_driver
@@ -2749,6 +2878,17 @@ module BaseTest
     _undefined
   end
 
+  # Compare the timestamp seconds and nanoseconds with the expected timestamp.
+  def assert_timestamp_matches(expected_ts, ts_secs, ts_nanos, entry)
+    assert_equal expected_ts.tv_sec, ts_secs, entry
+    # Fluentd v0.14 onwards supports nanosecond timestamp values.
+    # Added in 600 ns delta to avoid flaky tests introduced
+    # due to rounding error in double-precision floating-point numbers
+    # (to account for the missing 9 bits of precision ~ 512 ns).
+    # See http://wikipedia.org/wiki/Double-precision_floating-point_format.
+    assert_in_delta expected_ts.tv_nsec, ts_nanos, 600, entry
+  end
+
   def assert_prometheus_metric_value(metric_name, expected_value, labels = {})
     metric = Prometheus::Client.registry.get(metric_name)
     assert_not_nil(metric)

data/test/plugin/constants.rb

@@ -274,10 +274,6 @@ module Constants
     split_logs_by_tag true
   ).freeze
 
-  NO_ADJUST_TIMESTAMPS_CONFIG = %(
-    adjust_invalid_timestamps false
-  ).freeze
-
   ENABLE_PROMETHEUS_CONFIG = %(
     enable_monitoring true
     monitoring_type prometheus

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-google-cloud
 version: !ruby/object:Gem::Version
-  version: 0.7.28
+  version: 0.7.29
 platform: ruby
 authors:
 - Stackdriver Agents Team
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-01-07 00:00:00.000000000 Z
+date: 2020-01-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd