embulk-input-mixpanel 0.5.6 → 0.5.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 4244a94a3544327e669c32d73c2ccee585995447
-  data.tar.gz: 626163f5745eb15632ba40f53322a406e1aa2281
+  metadata.gz: da76e44b6dedb0753c503b715003e6c26a530321
+  data.tar.gz: bd708d46d831ddb872c501623675627754d988db
 SHA512:
-  metadata.gz: 4d5261a04ea1ec237282956ffd68423974ba3e84858a77e4fd7b86f4f86984453111e60d0671ad539131ebf435da160793fad7389f0e134652b4d19618de91f5
-  data.tar.gz: 4a427ea5b7f21463b6fc608afb9dc9d39d6ff1cd94383aabee10d7f89205fd43f87e611d2de1df2653e0afd869527b7866d9552e77d2a3562fa7c56ef5a7ef46
+  metadata.gz: b63e44eb7a0f39f2b72f20b44e2075a943ed770073fd9740e2a097397d1d0417c11558ab7b36eb66ced530500295ec15d5846d1f487554398722c4995036fd4a
+  data.tar.gz: 6310dd9990715a6ab9aacf3bfcc6e377b24ed03b3841e1aaae44a709c7b2c66483fb4ea1bdffee936e09f07b164eb1aecc5def51233baa7a9f0f9ceffc641e02

data/embulk-input-mixpanel.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |spec|
   spec.name          = "embulk-input-mixpanel"
-  spec.version       = "0.5.6"
+  spec.version       = "0.5.7"
   spec.authors       = ["yoshihara", "uu59"]
   spec.summary       = "Mixpanel input plugin for Embulk"
   spec.description   = "Loads records from Mixpanel."

data/lib/embulk/input/mixpanel.rb

@@ -66,7 +66,8 @@ module Embulk
           retry_limit: config.param(:retry_limit, :integer, default: 5),
           latest_fetched_time: latest_fetched_time,
           incremental: incremental,
-          slice_range: config.param(:slice_range, :integer, default: 7)
+          slice_range: config.param(:slice_range, :integer, default: 7),
+          job_start_time: Time.now.to_i*1000
         }
 
         if task[:fetch_unknown_columns] && task[:fetch_custom_properties]
@@ -159,13 +160,13 @@ module Embulk
         prev_latest_fetched_time = task[:latest_fetched_time] || 0
         prev_latest_fetched_time_format = Time.at(prev_latest_fetched_time).strftime("%F %T %z")
         current_latest_fetched_time = prev_latest_fetched_time
-        @dates.each_slice(task[:slice_range]) do |sliced_dates|
+        @dates.each_slice(task[:slice_range]) do |dates|
           ignored_record_count = 0
           unless preview?
-            Embulk.logger.info "Fetching data from #{@dates.first} to #{sliced_dates.last} ..."
+            Embulk.logger.info "Fetching data from #{dates.first} to #{dates.last} ..."
           end
           record_time_column=@incremental_column || DEFAULT_TIME_COLUMN
-          fetch([@dates.first, sliced_dates.last], prev_latest_fetched_time).each do |record|
+          fetch(dates, prev_latest_fetched_time).each do |record|
             if @incremental
               if !record["properties"].include?(record_time_column)
                 raise Embulk::ConfigError.new("Incremental column not exists in fetched data #{record_time_column}")
@@ -193,7 +194,6 @@ module Embulk
             end
             page_builder.add(values)
           end
-          prev_latest_fetched_time = [current_latest_fetched_time, prev_latest_fetched_time].max
           if ignored_record_count > 0
             Embulk.logger.warn "Skipped already loaded #{ignored_record_count} records. These record times are older or equal than previous fetched record time (#{prev_latest_fetched_time} @ #{prev_latest_fetched_time_format})."
           end
@@ -258,7 +258,7 @@ module Embulk
         end
       end
 
-      def fetch(dates,last_fetch_time, &block)
+      def fetch(dates, last_fetch_time, &block)
         from_date = dates.first
         to_date = dates.last
         params = @params.merge(
@@ -267,7 +267,7 @@ module Embulk
         )
         if !@incremental_column.nil? && !last_fetch_time.nil? && last_fetch_time!=0 # can't do filter on time column, time column need to be filter manually.
           params = params.merge(
-            "where" => "#{params['where'].nil? ? '' : "(#{params['where']}) and " }properties[\"#{@incremental_column}\"] > #{last_fetch_time}"
+            "where" => "#{params['where'].nil? ? '' : "(#{params['where']}) and " }properties[\"#{@incremental_column}\"] > #{last_fetch_time} and properties[\"#{@incremental_column}\"] < #{task[:job_start_time]}"
           )
         end
         Embulk.logger.info "Where params is #{params["where"]}"

data/test/embulk/input/test_mixpanel.rb

@@ -15,7 +15,7 @@ module Embulk
       DAYS = 8
       DATES = Date.parse(FROM_DATE)..(Date.parse(FROM_DATE) + DAYS - 1)
       TIMEZONE = "Asia/Tokyo".freeze
-
+      JOB_START_TIME = 1506407051000
       DURATIONS = [
         {from_date: FROM_DATE, to_date: "2015-02-28"}, # It has 7 days between 2015-02-22 and 2015-02-28
         {from_date: "2015-03-01", to_date: TO_DATE},
@@ -45,7 +45,13 @@ module Embulk
           end
         end
       end
-
+      def satisfy_task_ignore_start_time(expected_task)
+        satisfy{|input_task|
+          assert_not_nil(input_task[:job_start_time])
+          assert_equal(expected_task, input_task.merge(job_start_time: expected_task[:job_start_time]))
+          true
+        }
+      end
       def setup_logger
         stub(Embulk).logger { ::Logger.new(IO::NULL) }
       end
@@ -220,13 +226,12 @@ module Embulk
 
           def test_ignore_early_days
             stub(Embulk).logger { Logger.new(File::NULL) }
-
-            mock(Mixpanel).resume(task.merge(dates: target_dates), columns, 1, &control)
+            mock(Mixpanel).resume(satisfy_task_ignore_start_time(task.merge(dates: target_dates)), columns, 1, &control)
             Mixpanel.transaction(transaction_config, &control)
           end
 
           def test_info
-            stub(Mixpanel).resume(task.merge(dates: target_dates), columns, 1, &control)
+            stub(Mixpanel).resume(satisfy_task_ignore_start_time(task.merge(dates: target_dates)), columns, 1, &control)
 
             info_message_regexp = /#{Regexp.escape(target_dates.first)}.+#{Regexp.escape(target_dates.last)}/
             mock(Embulk.logger).info(info_message_regexp)
@@ -236,7 +241,7 @@ module Embulk
           end
 
           def test_warn
-            stub(Mixpanel).resume(task.merge(dates: target_dates), columns, 1, &control)
+            stub(Mixpanel).resume(satisfy_task_ignore_start_time(task.merge(dates: target_dates)), columns, 1, &control)
             stub(Embulk.logger).info
 
             ignore_dates = dates.map{|date| date.to_s}.to_a - target_dates
@@ -270,8 +275,7 @@ module Embulk
         class TimezoneTest < self
           def test_valid_timezone
             timezone = TIMEZONE
-            mock(Mixpanel).resume(transaction_task(timezone), columns, 1, &control)
-
+            mock(Mixpanel).resume(satisfy_task_ignore_start_time(transaction_task(timezone)), columns, 1, &control)
             Mixpanel.transaction(transaction_config(timezone), &control)
           end
 
@@ -310,7 +314,7 @@ module Embulk
           def test_valid_days
             days = 5
 
-            mock(Mixpanel).resume(transaction_task(days), columns, 1, &control)
+            mock(Mixpanel).resume(satisfy_task_ignore_start_time(transaction_task(days)), columns, 1, &control)
             Mixpanel.transaction(transaction_config(days), &control)
           end
 
@@ -541,7 +545,8 @@ module Embulk
             retry_initial_wait_sec: 0,
             retry_limit: 3,
             latest_fetched_time: 0,
-            slice_range: 7
+            slice_range: 7,
+            job_start_time: JOB_START_TIME
           }
         end
       end
@@ -571,7 +576,7 @@ module Embulk
 
         def test_run
           stub(@plugin).preview? { false }
-          mock(@page_builder).add(anything).times(records.length)
+          mock(@page_builder).add(anything).times(records.length * 2)
           mock(@page_builder).finish
 
           @plugin.run
@@ -580,7 +585,7 @@ module Embulk
         def test_timezone
           stub(@plugin).preview? { false }
           adjusted = record_epoch - timezone_offset_seconds
-          mock(@page_builder).add(["FOO", adjusted, "event"]).times(records.length)
+          mock(@page_builder).add(["FOO", adjusted, "event"]).times(records.length * 2)
           mock(@page_builder).finish
 
           @plugin.run
@@ -592,9 +597,9 @@ module Embulk
             plugin = Mixpanel.new(task.merge(slice_range: 2), nil, nil, @page_builder)
             stub(plugin).preview? {false}
             stub(plugin).fetch(["2015-02-22", "2015-02-23"],0){[]}
-            stub(plugin).fetch(["2015-02-22", "2015-02-25"],0){[]}
-            stub(plugin).fetch(["2015-02-22", "2015-02-27"],0){[]}
-            stub(plugin).fetch(["2015-02-22", "2015-03-01"],0){[]}
+            stub(plugin).fetch(["2015-02-24", "2015-02-25"],0){[]}
+            stub(plugin).fetch(["2015-02-26", "2015-02-27"],0){[]}
+            stub(plugin).fetch(["2015-02-28", "2015-03-01"],0){[]}
             mock(@page_builder).finish
             plugin.run
           end
@@ -644,14 +649,8 @@ module Embulk
             mock(page_builder).add(["FOO", adjusted, "event"]).times(records.length * 2)
             mock(page_builder).finish
             any_instance_of(MixpanelApi::Client) do |klass|
-              stub(klass).export(satisfy{|params|
-                '(abc==def) and properties["mp_processing_time_ms"] > 1'==params["where"]
-              }).once do |params, block|
-                records.each{|record| block.call(record) }
-              end
-              stub(klass).export(satisfy{|params|
-                '(abc==def) and properties["mp_processing_time_ms"] > 1234567919'==params["where"]
-              }).once do |params, block|
+              stub(klass).export() do |params, block|
+                assert_equal("(abc==def) and properties[\"mp_processing_time_ms\"] > 1 and properties[\"mp_processing_time_ms\"] < #{JOB_START_TIME}",params["where"])
                 records.each{|record| block.call(record) }
               end
             end
@@ -667,14 +666,8 @@ module Embulk
             mock(page_builder).add(["FOO", adjusted, "event"]).times(records.length * 2)
             mock(page_builder).finish
             any_instance_of(MixpanelApi::Client) do |klass|
-              stub(klass).export(satisfy {|params|
-                'properties["mp_processing_time_ms"] > 1'==params["where"]
-              }).once() do |params,block|
-                records.each{|record| block.call(record) }
-              end
-              stub(klass).export(satisfy {|params|
-                'properties["mp_processing_time_ms"] > 1234567919' == params["where"]
-              }).once() do |params,block|
+              stub(klass).export() do |params, block|
+                assert_equal("properties[\"mp_processing_time_ms\"] > 1 and properties[\"mp_processing_time_ms\"] < #{JOB_START_TIME}",params["where"])
                 records.each{|record| block.call(record) }
               end
             end
@@ -782,7 +775,7 @@ module Embulk
               {"int" => properties["int"], "event" => record["event"]}.to_json
             ]
 
-            mock(@page_builder).add(added).times(records.length)
+            mock(@page_builder).add(added).times(records.length * 2)
             mock(@page_builder).finish
 
             @plugin.run
@@ -834,7 +827,8 @@ module Embulk
           retry_initial_wait_sec: 2,
           retry_limit: 3,
           latest_fetched_time: 0,
-          slice_range: 7
+          slice_range: 7,
+          job_start_time: JOB_START_TIME
         }
       end
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: embulk-input-mixpanel
 version: !ruby/object:Gem::Version
-  version: 0.5.6
+  version: 0.5.7
 platform: ruby
 authors:
 - yoshihara
@@ -9,7 +9,7 @@ authors:
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2017-09-20 00:00:00.000000000 Z
+date: 2017-09-26 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -224,7 +224,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project:
-rubygems_version: 2.4.8
+rubygems_version: 2.6.6
 signing_key:
 specification_version: 4
 summary: Mixpanel input plugin for Embulk