embulk-input-mixpanel 0.5.5 → 0.5.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: feae04dbd0d3d9eebea301fecf14418a588723d4
-  data.tar.gz: 0ae3937206d959f34f1d542297925388f616f31b
+  metadata.gz: 4244a94a3544327e669c32d73c2ccee585995447
+  data.tar.gz: 626163f5745eb15632ba40f53322a406e1aa2281
 SHA512:
-  metadata.gz: 742c1956e206d8876f40bf564b40d072cf4f8416850e10584228a4f290a889a2171354b94ce23e2f1b84cbb28e3401e640ea4f4084648b43de0a0ac0ea91d158
-  data.tar.gz: 3827c9c15d5db544261a1c666286ed4a465d8caf42a1cce0ab21f55e7092c9322430df180ec0b08af645df3de271fe461885863388a2aa019ccbaa4a8d27de79
+  metadata.gz: 4d5261a04ea1ec237282956ffd68423974ba3e84858a77e4fd7b86f4f86984453111e60d0671ad539131ebf435da160793fad7389f0e134652b4d19618de91f5
+  data.tar.gz: 4a427ea5b7f21463b6fc608afb9dc9d39d6ff1cd94383aabee10d7f89205fd43f87e611d2de1df2653e0afd869527b7866d9552e77d2a3562fa7c56ef5a7ef46

data/CHANGELOG.md

@@ -1,3 +1,9 @@
+## 0.5.6 - 2017-09-20
+* [bug] Fix issue when back_fill data get processed by Mixpanel when plugin is running [#54](https://github.com/treasure-data/embulk-input-mixpanel/pull/54)
+
+## 0.5.5 - 2017-09-11
+* [enhancement] Add slice_range configuration [#52](https://github.com/treasure-data/embulk-input-mixpanel/pull/52)
+
 ## 0.5.4 - 2017-08-15
 * [bug] Fix a bug when `fetch_days` is 1 plugin will fetch 2 days instead of 1 [#51](https://github.com/treasure-data/embulk-input-mixpanel/pull/51)
 

data/embulk-input-mixpanel.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |spec|
   spec.name          = "embulk-input-mixpanel"
-  spec.version       = "0.5.5"
+  spec.version       = "0.5.6"
   spec.authors       = ["yoshihara", "uu59"]
   spec.summary       = "Mixpanel input plugin for Embulk"
   spec.description   = "Loads records from Mixpanel."

data/lib/embulk/input/mixpanel.rb

@@ -159,13 +159,13 @@ module Embulk
         prev_latest_fetched_time = task[:latest_fetched_time] || 0
         prev_latest_fetched_time_format = Time.at(prev_latest_fetched_time).strftime("%F %T %z")
         current_latest_fetched_time = prev_latest_fetched_time
-        @dates.each_slice(task[:slice_range]) do |dates|
+        @dates.each_slice(task[:slice_range]) do |sliced_dates|
           ignored_record_count = 0
           unless preview?
-            Embulk.logger.info "Fetching data from #{dates.first} to #{dates.last} ..."
+            Embulk.logger.info "Fetching data from #{@dates.first} to #{sliced_dates.last} ..."
           end
           record_time_column=@incremental_column || DEFAULT_TIME_COLUMN
-          fetch(dates, prev_latest_fetched_time).each do |record|
+          fetch([@dates.first, sliced_dates.last], prev_latest_fetched_time).each do |record|
             if @incremental
               if !record["properties"].include?(record_time_column)
                 raise Embulk::ConfigError.new("Incremental column not exists in fetched data #{record_time_column}")
@@ -193,6 +193,7 @@ module Embulk
             end
             page_builder.add(values)
           end
+          prev_latest_fetched_time = [current_latest_fetched_time, prev_latest_fetched_time].max
           if ignored_record_count > 0
             Embulk.logger.warn "Skipped already loaded #{ignored_record_count} records. These record times are older or equal than previous fetched record time (#{prev_latest_fetched_time} @ #{prev_latest_fetched_time_format})."
           end

data/test/embulk/input/test_mixpanel.rb

@@ -571,7 +571,7 @@ module Embulk
 
         def test_run
           stub(@plugin).preview? { false }
-          mock(@page_builder).add(anything).times(records.length * 2)
+          mock(@page_builder).add(anything).times(records.length)
           mock(@page_builder).finish
 
           @plugin.run
@@ -580,7 +580,7 @@ module Embulk
         def test_timezone
           stub(@plugin).preview? { false }
           adjusted = record_epoch - timezone_offset_seconds
-          mock(@page_builder).add(["FOO", adjusted, "event"]).times(records.length * 2)
+          mock(@page_builder).add(["FOO", adjusted, "event"]).times(records.length)
           mock(@page_builder).finish
 
           @plugin.run
@@ -592,9 +592,9 @@ module Embulk
             plugin = Mixpanel.new(task.merge(slice_range: 2), nil, nil, @page_builder)
             stub(plugin).preview? {false}
             stub(plugin).fetch(["2015-02-22", "2015-02-23"],0){[]}
-            stub(plugin).fetch(["2015-02-24", "2015-02-25"],0){[]}
-            stub(plugin).fetch(["2015-02-26", "2015-02-27"],0){[]}
-            stub(plugin).fetch(["2015-02-28", "2015-03-01"],0){[]}
+            stub(plugin).fetch(["2015-02-22", "2015-02-25"],0){[]}
+            stub(plugin).fetch(["2015-02-22", "2015-02-27"],0){[]}
+            stub(plugin).fetch(["2015-02-22", "2015-03-01"],0){[]}
             mock(@page_builder).finish
             plugin.run
           end
@@ -644,8 +644,14 @@ module Embulk
             mock(page_builder).add(["FOO", adjusted, "event"]).times(records.length * 2)
             mock(page_builder).finish
             any_instance_of(MixpanelApi::Client) do |klass|
-              stub(klass).export() do |params, block|
-                assert_equal('(abc==def) and properties["mp_processing_time_ms"] > 1',params["where"])
+              stub(klass).export(satisfy{|params|
+                '(abc==def) and properties["mp_processing_time_ms"] > 1'==params["where"]
+              }).once do |params, block|
+                records.each{|record| block.call(record) }
+              end
+              stub(klass).export(satisfy{|params|
+                '(abc==def) and properties["mp_processing_time_ms"] > 1234567919'==params["where"]
+              }).once do |params, block|
                 records.each{|record| block.call(record) }
               end
             end
@@ -661,8 +667,14 @@ module Embulk
             mock(page_builder).add(["FOO", adjusted, "event"]).times(records.length * 2)
             mock(page_builder).finish
             any_instance_of(MixpanelApi::Client) do |klass|
-              stub(klass).export() do |params, block|
-                assert_equal('properties["mp_processing_time_ms"] > 1',params["where"])
+              stub(klass).export(satisfy {|params|
+                'properties["mp_processing_time_ms"] > 1'==params["where"]
+              }).once() do |params,block|
+                records.each{|record| block.call(record) }
+              end
+              stub(klass).export(satisfy {|params|
+                'properties["mp_processing_time_ms"] > 1234567919' == params["where"]
+              }).once() do |params,block|
                 records.each{|record| block.call(record) }
               end
             end
@@ -770,7 +782,7 @@ module Embulk
               {"int" => properties["int"], "event" => record["event"]}.to_json
             ]
 
-            mock(@page_builder).add(added).times(records.length * 2)
+            mock(@page_builder).add(added).times(records.length)
             mock(@page_builder).finish
 
             @plugin.run

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: embulk-input-mixpanel
 version: !ruby/object:Gem::Version
-  version: 0.5.5
+  version: 0.5.6
 platform: ruby
 authors:
 - yoshihara
@@ -9,7 +9,7 @@ authors:
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2017-09-11 00:00:00.000000000 Z
+date: 2017-09-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -224,7 +224,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project:
-rubygems_version: 2.6.6
+rubygems_version: 2.4.8
 signing_key:
 specification_version: 4
 summary: Mixpanel input plugin for Embulk