embulk-input-mixpanel 0.5.4 → 0.5.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: f5341d3459ab9ebb11548c637781a2e56f8f827d
-  data.tar.gz: fddc31638fbfc95d804b140e91a79a880e2fd0c8
+  metadata.gz: feae04dbd0d3d9eebea301fecf14418a588723d4
+  data.tar.gz: 0ae3937206d959f34f1d542297925388f616f31b
 SHA512:
-  metadata.gz: ddd3e72b06554148f5bf7e866823aa3e65004ef317f7a55df269c41499515f8b23b63f4571c565bbe0123871bb1cc0d9451fc25f36939f1f5974737699152730
-  data.tar.gz: c59a6e927cb172562d0ecead325d4f8c5359f6bec34f672673abdf9a061b9e7aeecc617f820f99beffed30171fa038a8fa25a917f9bf82e923a49cb9a7405ac8
+  metadata.gz: 742c1956e206d8876f40bf564b40d072cf4f8416850e10584228a4f290a889a2171354b94ce23e2f1b84cbb28e3401e640ea4f4084648b43de0a0ac0ea91d158
+  data.tar.gz: 3827c9c15d5db544261a1c666286ed4a465d8caf42a1cce0ab21f55e7092c9322430df180ec0b08af645df3de271fe461885863388a2aa019ccbaa4a8d27de79

data/CHANGELOG.md

@@ -1,5 +1,5 @@
 ## 0.5.4 - 2017-08-15
-* [bug] Fix a bug when `fetch_days` is 1 plugin will fetch 2 days instead of 1
+* [bug] Fix a bug when `fetch_days` is 1 plugin will fetch 2 days instead of 1 [#51](https://github.com/treasure-data/embulk-input-mixpanel/pull/51)
 
 ## 0.5.3 - 2017-08-07
 * [enhancement] Allow user to choose to run incremental or not `incremental` option default to true [#50](https://github.com/treasure-data/embulk-input-mixpanel/pull/50)

data/embulk-input-mixpanel.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |spec|
   spec.name          = "embulk-input-mixpanel"
-  spec.version       = "0.5.4"
+  spec.version       = "0.5.5"
   spec.authors       = ["yoshihara", "uu59"]
   spec.summary       = "Mixpanel input plugin for Embulk"
   spec.description   = "Loads records from Mixpanel."

data/lib/embulk/input/mixpanel.rb

@@ -24,11 +24,8 @@ module Embulk
         $city $region mp_country_code $browser $browser_version $device $current_url $initial_referrer $initial_referring_domain $os $referrer $referring_domain $screen_height $screen_width $search_engine $city $region $mp_country_code $timezone $browser_version $browser $initial_referrer $initial_referring_domain $os $last_seen $city $region mp_country_code $app_release $app_version $carrier $ios_ifa $os_version $manufacturer $lib_version $model $os $screen_height $screen_width $wifi $city $region $mp_country_code $timezone $ios_app_release $ios_app_version $ios_device_model $ios_lib_version $ios_version $ios_ifa $last_seen $city $region mp_country_code $app_version $bluetooth_enabled $bluetooth_version $brand $carrier $has_nfc $has_telephone $lib_version $manufacturer $model $os $os_version $screen_dpi $screen_height $screen_width $wifi $google_play_services $city $region mp_country_code $timezone $android_app_version $android_app_version_code $android_lib_version $android_os $android_os_version $android_brand $android_model $android_manufacturer $last_seen
       ).uniq.freeze
 
-      # NOTE: It takes long time to fetch data between from_date to
-      # to_date by one API request. So this plugin fetches data
-      # between each 7 (SLICE_DAYS_COUNT) days.
-      SLICE_DAYS_COUNT = 7
 
+      DEFAULT_FETCH_DAYS = 7
       DEFAULT_TIME_COLUMN = 'time'
 
       def self.transaction(config, &control)
@@ -68,7 +65,8 @@ module Embulk
           incremental_column: incremental_column,
           retry_limit: config.param(:retry_limit, :integer, default: 5),
           latest_fetched_time: latest_fetched_time,
-          incremental: incremental
+          incremental: incremental,
+          slice_range: config.param(:slice_range, :integer, default: 7)
         }
 
         if task[:fetch_unknown_columns] && task[:fetch_custom_properties]
@@ -161,8 +159,7 @@ module Embulk
         prev_latest_fetched_time = task[:latest_fetched_time] || 0
         prev_latest_fetched_time_format = Time.at(prev_latest_fetched_time).strftime("%F %T %z")
         current_latest_fetched_time = prev_latest_fetched_time
-
-        @dates.each_slice(SLICE_DAYS_COUNT) do |dates|
+        @dates.each_slice(task[:slice_range]) do |dates|
           ignored_record_count = 0
           unless preview?
             Embulk.logger.info "Fetching data from #{dates.first} to #{dates.last} ..."
@@ -196,7 +193,6 @@ module Embulk
             end
             page_builder.add(values)
           end
-
           if ignored_record_count > 0
             Embulk.logger.warn "Skipped already loaded #{ignored_record_count} records. These record times are older or equal than previous fetched record time (#{prev_latest_fetched_time} @ #{prev_latest_fetched_time_format})."
           end
@@ -316,12 +312,12 @@ module Embulk
       end
 
       def self.default_guess_start_date
-        Date.today - SLICE_DAYS_COUNT - 1
+        Date.today - DEFAULT_FETCH_DAYS - 1
       end
 
       def self.guess_range(config)
         from_date = config.param(:from_date, :string, default: default_guess_start_date.to_s)
-        fetch_days = config.param(:fetch_days, :integer, default: SLICE_DAYS_COUNT)
+        fetch_days = config.param(:fetch_days, :integer, default: DEFAULT_FETCH_DAYS)
         range = RangeGenerator.new(from_date, fetch_days).generate_range
         if range.empty?
           return default_guess_start_date..(Date.today - 1)

data/test/embulk/input/test_mixpanel.rb

@@ -541,6 +541,7 @@ module Embulk
             retry_initial_wait_sec: 0,
             retry_limit: 3,
             latest_fetched_time: 0,
+            slice_range: 7
           }
         end
       end
@@ -585,6 +586,20 @@ module Embulk
           @plugin.run
         end
 
+        class SliceRangeRunTest < self
+
+          def test_default_slice_range
+            plugin = Mixpanel.new(task.merge(slice_range: 2), nil, nil, @page_builder)
+            stub(plugin).preview? {false}
+            stub(plugin).fetch(["2015-02-22", "2015-02-23"],0){[]}
+            stub(plugin).fetch(["2015-02-24", "2015-02-25"],0){[]}
+            stub(plugin).fetch(["2015-02-26", "2015-02-27"],0){[]}
+            stub(plugin).fetch(["2015-02-28", "2015-03-01"],0){[]}
+            mock(@page_builder).finish
+            plugin.run
+          end
+        end
+
         class NonIncrementalRunTest < self
 
           def test_non_incremental_run
@@ -807,6 +822,7 @@ module Embulk
           retry_initial_wait_sec: 2,
           retry_limit: 3,
           latest_fetched_time: 0,
+          slice_range: 7
         }
       end
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: embulk-input-mixpanel
 version: !ruby/object:Gem::Version
-  version: 0.5.4
+  version: 0.5.5
 platform: ruby
 authors:
 - yoshihara
@@ -9,7 +9,7 @@ authors:
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2017-08-15 00:00:00.000000000 Z
+date: 2017-09-11 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -224,7 +224,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project:
-rubygems_version: 2.6.12
+rubygems_version: 2.6.6
 signing_key:
 specification_version: 4
 summary: Mixpanel input plugin for Embulk