embulk-input-mixpanel 0.5.13 → 0.5.14

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 66b8edacf57562c25bb7663220b5c2f85e585da4
-  data.tar.gz: d631d41e6761ba3dc840085d24d9666a0f0b1221
+  metadata.gz: cf14d1cd9b35cd1dbde2a915a31fa71b7c321fc3
+  data.tar.gz: 3bd9b09fd763e0b42537f88879146606dbd92d1d
 SHA512:
-  metadata.gz: 38cdaa6706bf5ac3b76eb7be40a52ff2473b6e4fe3f7037f9d00ee61cfe35c94d0bd85a3a00d15a1d6486100c5ab1851aee1bce4c54cbeaadc93f90b2396523e
-  data.tar.gz: 0c778717481718ecf4b97c636ec0f5ed6966be18b8687ba0c95941e66726fd4fe543674490a266d5d9948ac31b775574e229030f4efd26cc1159e8e14d9b8acc
+  metadata.gz: 5c1b12301829f2d451013c0883a06d442fad5321c1b497d7d8761ffd6bd7313ada5fa2c2ede54752a56b2e17f0f324741a449831deb4112625d0200ba2448fd6
+  data.tar.gz: d7fd325a8a6f70c0ff61330f5f3f0f1733242f20f2c5349bd8491ebcfab9c78bbf1a6eb0e870027b5b9ffe61bb15f7fdc073dc979b30278853a9c3ade7894be3

data/CHANGELOG.md

@@ -1,3 +1,7 @@
+## 0.5.14 - 2018-10-22
+
+* [enhancement] Handle the wrong period during transition from standard to daylight saving time exception [#61](https://github.com/treasure-data/embulk-input-mixpanel/pull/61)
+
 ## 0.5.13 - 2018-10-04
 
 * [enhancement] Limit number of returned records in guess and preview [#60](https://github.com/treasure-data/embulk-input-mixpanel/pull/60)

data/embulk-input-mixpanel.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |spec|
   spec.name          = "embulk-input-mixpanel"
-  spec.version       = "0.5.13"
+  spec.version       = "0.5.14"
   spec.authors       = ["yoshihara", "uu59"]
   spec.summary       = "Mixpanel input plugin for Embulk"
   spec.description   = "Loads records from Mixpanel."

data/lib/embulk/input/mixpanel.rb

@@ -180,7 +180,13 @@ module Embulk
         prev_latest_fetched_time_format = Time.at(prev_latest_fetched_time).strftime("%F %T %z")
         current_latest_fetched_time = prev_latest_fetched_time
         @dates.each_slice(task[:slice_range]) do |slice_dates|
-          ignored_record_count = 0
+          ignored_fetched_record_count = 0
+          # There is the issue with Mixpanel time field during the transition from standard to daylight saving time
+          # in the US timezone i.e. 11 Mar 2018 2AM - 2:59AM, time within that period must not be existed,
+          # due to daylight saving, time will be forwarded 1 hour from 2AM to 3AM.
+          #
+          # All of records with wrong timezone will be ignored instead of throw exception out
+          ignored_wrong_daylight_tz_record_count = 0
           unless preview?
             Embulk.logger.info "Fetching data from #{slice_dates.first} to #{slice_dates.last} ..."
           end
@@ -194,7 +200,7 @@ module Embulk
                 record_time = record["properties"][record_time_column]
                 if @incremental_column.nil?
                   if record_time <= prev_latest_fetched_time
-                    ignored_record_count += 1
+                    ignored_fetched_record_count += 1
                     next
                   end
                 end
@@ -204,15 +210,19 @@ module Embulk
                   record_time,
                 ].max
               end
-              values = extract_values(record)
-              if @fetch_unknown_columns
-                unknown_values = extract_unknown_values(record)
-                values << unknown_values.to_json
-              end
-              if task[:fetch_custom_properties]
-                values << collect_custom_properties(record)
+              begin
+                values = extract_values(record)
+                if @fetch_unknown_columns
+                  unknown_values = extract_unknown_values(record)
+                  values << unknown_values.to_json
+                end
+                if task[:fetch_custom_properties]
+                  values << collect_custom_properties(record)
+                end
+                page_builder.add(values)
+              rescue TZInfo::PeriodNotFound
+                ignored_wrong_daylight_tz_record_count += 1
               end
-              page_builder.add(values)
             end
           rescue MixpanelApi::IncompleteExportResponseError
             if !task[:allow_partial_import]
@@ -220,8 +230,11 @@ module Embulk
               raise
             end
           end
-          if ignored_record_count > 0
-            Embulk.logger.warn "Skipped already loaded #{ignored_record_count} records. These record times are older or equal than previous fetched record time (#{prev_latest_fetched_time} @ #{prev_latest_fetched_time_format})."
+          if ignored_fetched_record_count > 0
+            Embulk.logger.warn "Skipped already loaded #{ignored_fetched_record_count} records. These record times are older or equal than previous fetched record time (#{prev_latest_fetched_time} @ #{prev_latest_fetched_time_format})."
+          end
+          if ignored_wrong_daylight_tz_record_count > 0
+            Embulk.logger.warn "Skipped #{ignored_wrong_daylight_tz_record_count} records due to corrupted Mixpanel time transition from standard to daylight saving"
           end
           break if preview?
         end
@@ -314,7 +327,7 @@ module Embulk
         # Adjust timezone offset to get UTC time
         # c.f. https://mixpanel.com/docs/api-documentation/exporting-raw-data-you-inserted-into-mixpanel#export
         tz = TZInfo::Timezone.get(@timezone)
-        offset = tz.period_for_local(epoch, true).offset.utc_offset
+        offset = tz.period_for_local(epoch, true).offset.utc_total_offset
         epoch - offset
       end
 

data/test/embulk/input/test_mixpanel.rb

@@ -798,7 +798,7 @@ module Embulk
 
             time = properties["time"]
             tz = TZInfo::Timezone.get(TIMEZONE)
-            offset = tz.period_for_local(time, true).offset.utc_offset
+            offset = tz.period_for_local(time, true).offset.utc_total_offset
             adjusted_time = time - offset
 
             added = [

data/test/test_range_generator.rb

@@ -5,7 +5,6 @@ require "active_support/core_ext/time"
 class RangeGeneratorTest < Test::Unit::TestCase
   include OverrideAssertRaise
   DEFAULT_TIMEZONE = "America/Chicago"
-  DEFAULT_LOCAL = ActiveSupport::TimeZone["UTC"]
   class GenerateRangeTest < self
     data do
       {
@@ -114,7 +113,7 @@ class RangeGeneratorTest < Test::Unit::TestCase
       RangeGenerator.new(from_date_str, fetch_days, DEFAULT_TIMEZONE).generate_range
     end
     def today
-      DEFAULT_LOCAL.today
+      ActiveSupport::TimeZone[DEFAULT_TIMEZONE].today
     end
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: embulk-input-mixpanel
 version: !ruby/object:Gem::Version
-  version: 0.5.13
+  version: 0.5.14
 platform: ruby
 authors:
 - yoshihara
@@ -9,7 +9,7 @@ authors:
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2018-10-04 00:00:00.000000000 Z
+date: 2018-10-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement