embulk-input-mixpanel 0.5.8 → 0.5.9

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 918d88c52e1dd983b7d78dfa673ce5f946ef0629
-  data.tar.gz: 9d17d52d3c3daf84f3b11d6b068ff2b873627ab7
+  metadata.gz: 71b5e68b20fb92e74d3a3210ee1700f90bdf26bc
+  data.tar.gz: b29ffe1fdf6f9f759cb6d0790e1448da6fd34ec2
 SHA512:
-  metadata.gz: 235b268d682ff9a4a964d37455ea873cd331b78bd9a2d44e7053b584e7910d5da4f5a2087c9ce9497d30fc06672e1956ce94f8a64885bdbbbd38c0bf7cfbca97
-  data.tar.gz: bb8c5c8808e95b636c41ae5f72f0f8135390face7765b3fbb6d7f3940da95dbaf6e6f034541597b926abfc2f63efa56eef7c14069d36015dbc90ec490a646e34
+  metadata.gz: 527ebe18c1cf8a7e6f8ab553f4b272f1719c09c8df4f2a3530f2ba267b7d633056cb6641b9681117f4fb2497a1cf4265ede7cce75b5f6e947e24b5333b55fd5d
+  data.tar.gz: 279e5750a346ed4b04c104418d56b7b7e7427d5a580ad1f7481e124b5ba893d4c370eb27577686db9fa11f6d1f18d513f22504441353a4547082356163a5d1be

data/CHANGELOG.md

@@ -1,7 +1,11 @@
+## 0.5.9 - 2017-11-10
+
+* [enhancement] Add upper limit delay to incremental column query [#56](https://github.com/treasure-data/embulk-input-mixpanel/pull/56)
+
 ## 0.5.8 - 2017-09-26
 
-* [bug] Fix issue when back_fill data get processed by Mixpanel when plugin is running [#56](https://github.com/treasure-data/embulk-input-mixpanel/pull/56)
-* [bug] Revert PR #54 [#56](https://github.com/treasure-data/embulk-input-mixpanel/pull/56)
+* [bug] Fix issue when back_fill data get processed by Mixpanel when plugin is running
+* [bug] Revert PR #54
 
 ## 0.5.6 - 2017-09-20
 * [bug] Fix issue when back_fill data get processed by Mixpanel when plugin is running [#54](https://github.com/treasure-data/embulk-input-mixpanel/pull/54)

data/embulk-input-mixpanel.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |spec|
   spec.name          = "embulk-input-mixpanel"
-  spec.version       = "0.5.8"
+  spec.version       = "0.5.9"
   spec.authors       = ["yoshihara", "uu59"]
   spec.summary       = "Mixpanel input plugin for Embulk"
   spec.description   = "Loads records from Mixpanel."

data/lib/embulk/input/mixpanel.rb

@@ -52,6 +52,9 @@ module Embulk
 
         range = RangeGenerator.new(from_date, fetch_days).generate_range
         Embulk.logger.info "Try to fetch data from #{range.first} to #{range.last}"
+        job_start_time = Time.now.to_i*1000
+        upper_limit_delay = config.param(:incremental_column_upper_limit_delay_in_seconds, :integer, default: 0)
+        incremental_column_upper_limit = job_start_time - (upper_limit_delay * 1000)
         task = {
           params: export_params(config),
           dates: range,
@@ -67,9 +70,14 @@ module Embulk
           latest_fetched_time: latest_fetched_time,
           incremental: incremental,
           slice_range: config.param(:slice_range, :integer, default: 7),
-          job_start_time: Time.now.to_i*1000
+          job_start_time: job_start_time,
+          incremental_column_upper_limit: incremental_column_upper_limit
         }
 
+        if !incremental_column.nil? && !latest_fetched_time.nil? && (incremental_column_upper_limit <= latest_fetched_time)
+            raise Embulk::ConfigError.new("Incremental column upper limit (job_start_time - incremental_column_upper_limit_delay_in_seconds) can't be smaller or equal latest fetched time #{latest_fetched_time}")
+        end
+
         if task[:fetch_unknown_columns] && task[:fetch_custom_properties]
           raise Embulk::ConfigError.new("Don't set true both `fetch_unknown_columns` and `fetch_custom_properties`.")
         end
@@ -156,6 +164,7 @@ module Embulk
       end
 
       def run
+        Embulk.logger.info "Job start time is #{task[:job_start_time]}"
         self.class.giveup_when_mixpanel_is_down
         prev_latest_fetched_time = task[:latest_fetched_time] || 0
         prev_latest_fetched_time_format = Time.at(prev_latest_fetched_time).strftime("%F %T %z")
@@ -267,7 +276,7 @@ module Embulk
         )
         if !@incremental_column.nil? # can't do filter on time column, time column need to be filter manually.
           params = params.merge(
-            "where" => "#{params['where'].nil? ? '' : "(#{params['where']}) and " }properties[\"#{@incremental_column}\"] > #{last_fetch_time || 0} and properties[\"#{@incremental_column}\"] < #{task[:job_start_time]}"
+            "where" => "#{params['where'].nil? ? '' : "(#{params['where']}) and " }properties[\"#{@incremental_column}\"] > #{last_fetch_time || 0} and properties[\"#{@incremental_column}\"] < #{task[:incremental_column_upper_limit]}"
           )
         end
         Embulk.logger.info "Where params is #{params["where"]}"

data/test/embulk/input/test_mixpanel.rb

@@ -16,6 +16,7 @@ module Embulk
       DATES = Date.parse(FROM_DATE)..(Date.parse(FROM_DATE) + DAYS - 1)
       TIMEZONE = "Asia/Tokyo".freeze
       JOB_START_TIME = 1506407051000
+      UPPER_LIMIT_DELAY = 300 # 300 seconds delay
       DURATIONS = [
         {from_date: FROM_DATE, to_date: "2015-02-28"}, # It has 7 days between 2015-02-22 and 2015-02-28
         {from_date: "2015-03-01", to_date: TO_DATE},
@@ -46,9 +47,13 @@ module Embulk
         end
       end
       def satisfy_task_ignore_start_time(expected_task)
-        satisfy{|input_task|
+        satisfy {|input_task|
           assert_not_nil(input_task[:job_start_time])
-          assert_equal(expected_task, input_task.merge(job_start_time: expected_task[:job_start_time]))
+          # This will also verify incremental column upper limit
+          assert_equal(input_task[:incremental_column_upper_limit], input_task[:job_start_time] - UPPER_LIMIT_DELAY * 1000)
+          expected_task[:job_start_time] = input_task[:job_start_time]
+          expected_task[:incremental_column_upper_limit] = input_task[:incremental_column_upper_limit]
+          assert_equal(expected_task, input_task)
           true
         }
       end
@@ -650,7 +655,7 @@ module Embulk
             mock(page_builder).finish
             any_instance_of(MixpanelApi::Client) do |klass|
               stub(klass).export() do |params, block|
-                assert_equal("(abc==def) and properties[\"mp_processing_time_ms\"] > 1 and properties[\"mp_processing_time_ms\"] < #{JOB_START_TIME}",params["where"])
+                assert_equal("(abc==def) and properties[\"mp_processing_time_ms\"] > 1 and properties[\"mp_processing_time_ms\"] < #{JOB_START_TIME - UPPER_LIMIT_DELAY * 1000}",params["where"])
                 records.each{|record| block.call(record) }
               end
             end
@@ -667,7 +672,7 @@ module Embulk
             mock(page_builder).finish
             any_instance_of(MixpanelApi::Client) do |klass|
               stub(klass).export() do |params, block|
-                assert_equal("properties[\"mp_processing_time_ms\"] > 1 and properties[\"mp_processing_time_ms\"] < #{JOB_START_TIME}",params["where"])
+                assert_equal("properties[\"mp_processing_time_ms\"] > 1 and properties[\"mp_processing_time_ms\"] < #{JOB_START_TIME - UPPER_LIMIT_DELAY * 1000}",params["where"])
                 records.each{|record| block.call(record) }
               end
             end
@@ -828,7 +833,8 @@ module Embulk
           retry_limit: 3,
           latest_fetched_time: 0,
           slice_range: 7,
-          job_start_time: JOB_START_TIME
+          job_start_time: JOB_START_TIME,
+          incremental_column_upper_limit: (JOB_START_TIME - UPPER_LIMIT_DELAY * 1000)
         }
       end
 
@@ -865,6 +871,7 @@ module Embulk
           retry_initial_wait_sec: 2,
           retry_limit: 3,
           latest_fetched_time: 0,
+          incremental_column_upper_limit_delay_in_seconds: UPPER_LIMIT_DELAY
         }
       end
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: embulk-input-mixpanel
 version: !ruby/object:Gem::Version
-  version: 0.5.8
+  version: 0.5.9
 platform: ruby
 authors:
 - yoshihara
@@ -9,79 +9,80 @@ authors:
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2017-09-26 00:00:00.000000000 Z
+date: 2017-11-10 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
+  name: httpclient
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: 2.8.3
-  name: httpclient
-  prerelease: false
-  type: :runtime
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: 2.8.3
+  prerelease: false
+  type: :runtime
 - !ruby/object:Gem::Dependency
+  name: tzinfo
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-  name: tzinfo
-  prerelease: false
-  type: :runtime
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+  prerelease: false
+  type: :runtime
 - !ruby/object:Gem::Dependency
+  name: perfect_retry
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '0.5'
-  name: perfect_retry
-  prerelease: false
-  type: :runtime
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '0.5'
+  prerelease: false
+  type: :runtime
 - !ruby/object:Gem::Dependency
+  name: bundler
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '1.0'
-  name: bundler
-  prerelease: false
-  type: :development
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '1.0'
+  prerelease: false
+  type: :development
 - !ruby/object:Gem::Dependency
+  name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: '10.0'
-  name: rake
-  prerelease: false
-  type: :development
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: '10.0'
+  prerelease: false
+  type: :development
 - !ruby/object:Gem::Dependency
+  name: embulk
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -90,9 +91,6 @@ dependencies:
     - - "<"
       - !ruby/object:Gem::Version
         version: '1.0'
-  name: embulk
-  prerelease: false
-  type: :development
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -101,76 +99,78 @@ dependencies:
     - - "<"
       - !ruby/object:Gem::Version
         version: '1.0'
+  prerelease: false
+  type: :development
 - !ruby/object:Gem::Dependency
+  name: pry
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-  name: pry
-  prerelease: false
-  type: :development
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+  prerelease: false
+  type: :development
 - !ruby/object:Gem::Dependency
+  name: test-unit
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-  name: test-unit
-  prerelease: false
-  type: :development
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+  prerelease: false
+  type: :development
 - !ruby/object:Gem::Dependency
+  name: test-unit-rr
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-  name: test-unit-rr
-  prerelease: false
-  type: :development
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+  prerelease: false
+  type: :development
 - !ruby/object:Gem::Dependency
+  name: codeclimate-test-reporter
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '0.5'
-  name: codeclimate-test-reporter
-  prerelease: false
-  type: :development
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '0.5'
+  prerelease: false
+  type: :development
 - !ruby/object:Gem::Dependency
+  name: everyleaf-embulk_helper
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-  name: everyleaf-embulk_helper
-  prerelease: false
-  type: :development
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+  prerelease: false
+  type: :development
 description: Loads records from Mixpanel.
 email:
 - h.yoshihara@everyleaf.com