embulk-input-mixpanel 0.5.11 → 0.5.12.alpha.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 6595e8e2cafe56340ff599d371303c45b868d70c
-  data.tar.gz: 96c82be9851fc2008b04a352294a7ad4e60ebcb9
+  metadata.gz: 62f37b674cb82550f2961ccf1e13ace51ea57fc2
+  data.tar.gz: b62127bec5d8c43b7998fdfce6f2dc560c29b343
 SHA512:
-  metadata.gz: ae2d8ad4e060e65a1ec37ba02e95603df8157448f3deeb04c8851ec1c2f2f4229e228b0a1cfbf74f4633a939adc1319d79404742d39543f75e879a8372210f76
-  data.tar.gz: e27412b27786522a2e3e2419f52be463cea2c7993814b0a375cce524901a6139ca1f7f27923746e04c54f6bfa8516edc75c23548135360f1e2375fe2ce7af248
+  metadata.gz: 240e77ee4ea62b38bb9f2cec3fb0a12019d16852efd4976106ea5175fd2d4121a18ebb4ed794ca27ee91ef999afe2ec0d19a1eaf7da2ccdb628c09aa28b1cdd1
+  data.tar.gz: eb44642ae282601ea8aea26067ad226b4235fa3b06bdeb35378d8ca19ee53406349bd6110a419cb67240120dc2f78b5d3ca8ebf50c853779a8246839b3101e85

data/embulk-input-mixpanel.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |spec|
   spec.name          = "embulk-input-mixpanel"
-  spec.version       = "0.5.11"
+  spec.version       = "0.5.12.alpha.1"
   spec.authors       = ["yoshihara", "uu59"]
   spec.summary       = "Mixpanel input plugin for Embulk"
   spec.description   = "Loads records from Mixpanel."
@@ -15,6 +15,7 @@ Gem::Specification.new do |spec|
   spec.add_dependency 'httpclient', '>= 2.8.3' # To use tcp_keepalive
   spec.add_dependency 'tzinfo'
   spec.add_dependency 'perfect_retry', ["~> 0.5"]
+  spec.add_dependency 'activesupport'
   spec.add_development_dependency 'bundler', ['~> 1.0']
   spec.add_development_dependency 'rake', ['>= 10.0']
   spec.add_development_dependency 'embulk', ['>= 0.8.6', '< 1.0']

data/lib/embulk/input/mixpanel.rb

@@ -4,6 +4,7 @@ require "embulk/input/mixpanel_api/client"
 require "embulk/input/mixpanel_api/exceptions"
 require "range_generator"
 require "timezone_validator"
+require "active_support/core_ext/time"
 
 module Embulk
   module Input
@@ -33,7 +34,7 @@ module Embulk
         timezone = config.param(:timezone, :string)
         TimezoneValidator.new(timezone).validate
 
-        from_date = config.param(:from_date, :string, default: (Date.today - 2).to_s)
+        from_date = config.param(:from_date, :string, default: (today(timezone) - 2).to_s)
         fetch_days = config.param(:fetch_days, :integer, default: nil)
 
 
@@ -51,7 +52,7 @@ module Embulk
           fetch_days = fetch_days.nil? ? nil : fetch_days + back_fill_days
         end
 
-        range = RangeGenerator.new(from_date, fetch_days).generate_range
+        range = RangeGenerator.new(from_date, fetch_days, timezone).generate_range
         Embulk.logger.info "Try to fetch data from #{range.first} to #{range.last}"
         job_start_time = Time.now.to_i*1000
         upper_limit_delay = config.param(:incremental_column_upper_limit_delay_in_seconds, :integer, default: 0)
@@ -228,7 +229,7 @@ module Embulk
         page_builder.finish
         task_report = {
           latest_fetched_time: current_latest_fetched_time,
-          to_date: @dates.last || Date.today - 1,
+          to_date: @dates.last || today(@timezone) - 1,
         }
         task_report
       end
@@ -338,16 +339,17 @@ module Embulk
         }
       end
 
-      def self.default_guess_start_date
-        Date.today - DEFAULT_FETCH_DAYS - 1
+      def self.default_guess_start_date(timezone)
+        today(timezone) - DEFAULT_FETCH_DAYS - 1
       end
 
       def self.guess_range(config)
-        from_date = config.param(:from_date, :string, default: default_guess_start_date.to_s)
+        time_zone = config.param(:timezone, :string, default: "")
+        from_date = config.param(:from_date, :string, default: default_guess_start_date(time_zone).to_s)
         fetch_days = config.param(:fetch_days, :integer, default: DEFAULT_FETCH_DAYS)
-        range = RangeGenerator.new(from_date, fetch_days).generate_range
+        range = RangeGenerator.new(from_date, fetch_days, time_zone).generate_range
         if range.empty?
-          return default_guess_start_date..(Date.today - 1)
+          return default_guess_start_date(time_zone)..(today(time_zone) - 1)
         end
         range
       end
@@ -368,7 +370,16 @@ module Embulk
         # Shift incremental column to top
         columns.unshift(name: "time", type: :long)
       end
-    end
 
+      def self.today(timezone)
+        if timezone.nil?
+          Date.today
+        else
+          zone = ActiveSupport::TimeZone[timezone]
+          zone.nil? ? Date.today : zone.today
+        end
+      end
+
+    end
   end
 end

data/lib/range_generator.rb

@@ -1,9 +1,12 @@
+require "active_support/core_ext/time"
+
 class RangeGenerator
-  attr_reader :from_date_str, :fetch_days
+  attr_reader :from_date_str, :fetch_days, :profile_time_zone
 
-  def initialize(from_date_str, fetch_days)
+  def initialize(from_date_str, fetch_days, profile_time_zone)
     @from_date_str = from_date_str
     @fetch_days = fetch_days
+    @profile_time_zone = profile_time_zone
   end
 
   def generate_range
@@ -70,6 +73,8 @@ class RangeGenerator
   end
 
   def today
-    @today ||= Date.today
+    zone = ActiveSupport::TimeZone[@profile_time_zone]
+    Embulk.logger.warn "Coudn't find timezone support for #{@profile_time_zone}" if zone.nil?
+    @today ||= (zone.nil? ? Date.today : zone.today)
   end
 end

data/test/embulk/input/test_mixpanel.rb

@@ -1,6 +1,7 @@
 require "prepare_embulk"
 require "override_assert_raise"
 require "embulk/input/mixpanel"
+require "active_support/core_ext/time"
 require "json"
 
 module Embulk
@@ -88,17 +89,18 @@ module Embulk
             type: "mixpanel",
             api_key: API_KEY,
             api_secret: API_SECRET,
-            from_date: (Date.today + 1).to_s,
+            timezone: TIMEZONE,
+            from_date: (today + 1).to_s
           }
 
           stub_export_all
-          mock(Embulk.logger).info(/Guessing.*#{Regexp.escape Mixpanel.default_guess_start_date.to_s}/)
+          mock(Embulk.logger).info(/Guessing.*#{Regexp.escape Mixpanel.default_guess_start_date(TIMEZONE).to_s}/)
 
           Mixpanel.guess(embulk_config(config))
         end
 
         def test_from_date_yesterday
-          from_date = (Date.today - 1).to_s
+          from_date = (today - 1).to_s
           config = {
             type: "mixpanel",
             api_key: API_KEY,
@@ -117,10 +119,11 @@ module Embulk
             type: "mixpanel",
             api_key: API_KEY,
             api_secret: API_SECRET,
+            timezone: TIMEZONE
           }
 
           stub_export_all
-          mock(Embulk.logger).info(/Guessing.*#{Regexp.escape Mixpanel.default_guess_start_date.to_s}/)
+          mock(Embulk.logger).info(/Guessing.*#{Regexp.escape Mixpanel.default_guess_start_date(TIMEZONE).to_s}/)
 
           Mixpanel.guess(embulk_config(config))
         end
@@ -192,7 +195,7 @@ module Embulk
         end
 
         def test_future
-          from_date = (Date.today + 10).to_s
+          from_date = (today + 10).to_s
           mock(Mixpanel).resume(anything, anything, 1)
 
           Mixpanel.transaction(transaction_config(from_date))
@@ -200,7 +203,7 @@ module Embulk
 
         def test_negative_days
           assert_raise(Embulk::ConfigError) do
-            Mixpanel.transaction(transaction_config((Date.today - 1).to_s).merge(fetch_days: -1))
+            Mixpanel.transaction(transaction_config((today - 1).to_s).merge(fetch_days: -1))
           end
         end
 
@@ -209,7 +212,7 @@ module Embulk
             assert_nil(task[:incremental_column])
             assert_true(task[:incremental])
           }
-          Mixpanel.transaction(transaction_config(Date.today))
+          Mixpanel.transaction(transaction_config(today))
         end
 
         private
@@ -259,11 +262,11 @@ module Embulk
           private
 
           def dates
-            (Date.today - 10)..(Date.today + 10)
+            (today - 10)..(today + 10)
           end
 
           def target_dates
-            dates.find_all{|d| d <= Date.today}.map {|date| date.to_s}
+            dates.find_all{|d| d <= today}.map {|date| date.to_s}
           end
 
           def transaction_config
@@ -325,9 +328,9 @@ module Embulk
 
           def test_next_to_date
             next_config_diff = Mixpanel.resume(transaction_task(1).merge(incremental: true), columns, 1) do
-              [{to_date: Date.today.to_s, latest_fetched_time: 1502707247000}]
+              [{to_date: today.to_s, latest_fetched_time: 1502707247000}]
             end
-            assert_equal(Date.today.to_s, next_config_diff[:from_date])
+            assert_equal(today.to_s, next_config_diff[:from_date])
           end
 
           def test_valid_days_with_backfill
@@ -419,7 +422,6 @@ module Embulk
         end
 
         def test_resume
-          today = Date.today
           control = proc { [{to_date: today.to_s, latest_fetched_time: 999}] }
           actual = Mixpanel.resume(transaction_task, columns, 1, &control)
           assert_equal({from_date: today.to_s, latest_fetched_time: 999}, actual)
@@ -905,6 +907,9 @@ module Embulk
       def embulk_config
         DataSource[*config.to_a.flatten(1)]
       end
+      def today
+        ActiveSupport::TimeZone[TIMEZONE].today
+      end
     end
   end
 end

data/test/test_range_generator.rb

@@ -1,9 +1,11 @@
 require "range_generator"
 require "override_assert_raise"
+require "active_support/core_ext/time"
 
 class RangeGeneratorTest < Test::Unit::TestCase
   include OverrideAssertRaise
-
+  DEFAULT_TIMEZONE = "America/Chicago"
+  DEFAULT_LOCAL = ActiveSupport::TimeZone["UTC"]
   class GenerateRangeTest < self
     data do
       {
@@ -25,7 +27,7 @@ class RangeGeneratorTest < Test::Unit::TestCase
 
       expected = (expected_from..expected_to).to_a.map{|date| date.to_s}
 
-      actual = RangeGenerator.new(from, days).generate_range
+      actual = RangeGenerator.new(from, days, DEFAULT_TIMEZONE).generate_range
 
       assert_equal(expected, actual)
     end
@@ -36,7 +38,7 @@ class RangeGeneratorTest < Test::Unit::TestCase
       expected_from = Date.parse(from)
       expected_to = Date.parse("2017-08-04")
       expected = (expected_from..expected_to).to_a.map {|date| date.to_s}
-      actual = RangeGenerator.new(from, days).generate_range
+      actual = RangeGenerator.new(from, days, DEFAULT_TIMEZONE).generate_range
       assert_equal(expected, actual)
     end
 
@@ -46,19 +48,20 @@ class RangeGeneratorTest < Test::Unit::TestCase
       expected_from = Date.parse(from)
       expected_to = Date.parse("2017-08-08")
       expected = (expected_from..expected_to).to_a.map {|date| date.to_s}
-      actual = RangeGenerator.new(from, days).generate_range
+      actual = RangeGenerator.new(from, days, DEFAULT_TIMEZONE).generate_range
       assert_equal(expected, actual)
     end
 
     class OverDaysTest < self
       def setup
-        @from = Date.today - 5
+        @from = today - 5
         @days = 10
         @warn_message_regexp = /ignored them/
       end
 
       def test_range_only_present
-        expected_to = Date.today
+        
+        expected_to = today
         expected = (@from..expected_to).to_a.map{|date| date.to_s}
 
         stub(Embulk.logger).warn(@warn_message_regexp)
@@ -81,7 +84,7 @@ class RangeGeneratorTest < Test::Unit::TestCase
 
     class FromDateEarlyTest < self
       def setup
-        @from = Date.today + 5
+        @from = today + 5
         @days = 10
         @warn_message_regexp = /allow 2 days/
       end
@@ -108,7 +111,10 @@ class RangeGeneratorTest < Test::Unit::TestCase
     private
 
     def generate_range(from_date_str, fetch_days)
-      RangeGenerator.new(from_date_str, fetch_days).generate_range
+      RangeGenerator.new(from_date_str, fetch_days, DEFAULT_TIMEZONE).generate_range
+    end
+    def today
+      DEFAULT_LOCAL.today
     end
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: embulk-input-mixpanel
 version: !ruby/object:Gem::Version
-  version: 0.5.11
+  version: 0.5.12.alpha.1
 platform: ruby
 authors:
 - yoshihara
@@ -9,7 +9,7 @@ authors:
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2017-12-11 00:00:00.000000000 Z
+date: 2018-02-05 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -53,6 +53,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '0.5'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  name: activesupport
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
@@ -220,12 +234,12 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">="
+  - - ">"
     - !ruby/object:Gem::Version
-      version: '0'
+      version: 1.3.1
 requirements: []
 rubyforge_project:
-rubygems_version: 2.4.8
+rubygems_version: 2.6.6
 signing_key:
 specification_version: 4
 summary: Mixpanel input plugin for Embulk