embulk-input-mixpanel 0.5.9 → 0.5.10

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 71b5e68b20fb92e74d3a3210ee1700f90bdf26bc
-  data.tar.gz: b29ffe1fdf6f9f759cb6d0790e1448da6fd34ec2
+  metadata.gz: 2db94c13e223960ee53abf712889a290728870d4
+  data.tar.gz: bae22b3a844dd7b9a3e0aa61cbb7c2c06846137f
 SHA512:
-  metadata.gz: 527ebe18c1cf8a7e6f8ab553f4b272f1719c09c8df4f2a3530f2ba267b7d633056cb6641b9681117f4fb2497a1cf4265ede7cce75b5f6e947e24b5333b55fd5d
-  data.tar.gz: 279e5750a346ed4b04c104418d56b7b7e7427d5a580ad1f7481e124b5ba893d4c370eb27577686db9fa11f6d1f18d513f22504441353a4547082356163a5d1be
+  metadata.gz: 72410d007e0d2035ecbeb1827d226c127fe62434f0e94cf3aa683ca4c1c6ba13c9107a1fdbc03d74dae00da7356034787f3da32417539a26efe0cf42f845278f
+  data.tar.gz: 65c4d423bd208e80a9d72f17d6289cf2369c7b740ecbd7397fafa219672b48ff7bdd59a58953051a4919645bc8f2963c64445edad25ea29eb8b2e490130f491f

data/CHANGELOG.md

@@ -1,3 +1,7 @@
+## 0.5.10 - 2017-12-03
+
+* [enhancement] Add logic to detect error from Mixpanel when doing import, add option to failed embulk job if encounter error import [#57](https://github.com/treasure-data/embulk-input-mixpanel/pull/57)
+
 ## 0.5.9 - 2017-11-10
 
 * [enhancement] Add upper limit delay to incremental column query [#56](https://github.com/treasure-data/embulk-input-mixpanel/pull/56)

data/README.md

@@ -42,6 +42,8 @@ To get it, you should log in mixpanel website, and click gear icon at the lower
 - **incremental_column**: Column to be add to where query as a constraint for incremental time. Only data that have incremental_column timestamp > than previous latest_fetched_time will be return (string, optional, default: nil)
 - **back_fill_time**: Amount of time that will be subtracted from `from_date` to calculate the final `from_date` that will be use for API Request. This is due to Mixpanel caching data on user devices before sending it to Mixpanel server (integer, optional, default: 5)
   - NOTE: Only have effect when incremental is true and incremental_column is specified
+- **incremental_column_upper_limit_delay_in_seconds**: When query with incremental column, plugin will lock the upper limit of incremental column query with the job start time, in order to avoid issue with data that commit when the job is running
+ ex: `where mp_processing_time <= job_start_time`. The upper limit will be calculated by using job_start_time minus with this configuration parameter. This is to support case when Mixpanel have delay in their processing (integer, optional, default: 0)
 - **fetch_unknown_columns**(deprecated): If you want this plugin fetches unknown (unconfigured in config) columns (boolean, optional, default: false)
   - NOTE: If true, `unknown_columns` column is created and added unknown columns' data.
 - **fetch_custom_properties**: All custom properties into `custom_properties` key. "custom properties" are not desribed Mixpanel document [1](https://mixpanel.com/help/questions/articles/special-or-reserved-properties), [2](https://mixpanel.com/help/questions/articles/what-properties-do-mixpanels-libraries-store-by-default).  (boolean, optional, default: true)
@@ -51,6 +53,7 @@ To get it, you should log in mixpanel website, and click gear icon at the lower
 - **bucket**:The data backet to filter data (string, optional, default: nil)
 - **retry_initial_wait_sec** Wait seconds for exponential backoff initial value (integer, default: 1)
 - **retry_limit**: Try to retry this times (integer, default: 5)
+- **allow_partial_import**: Allow plugin to skip errored import (boolean, default: true)
 
 ### `fetch_unknown_columns` and `fetch_custom_properties`
 

data/embulk-input-mixpanel.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |spec|
   spec.name          = "embulk-input-mixpanel"
-  spec.version       = "0.5.9"
+  spec.version       = "0.5.10"
   spec.authors       = ["yoshihara", "uu59"]
   spec.summary       = "Mixpanel input plugin for Embulk"
   spec.description   = "Loads records from Mixpanel."

data/lib/embulk/input/mixpanel.rb

@@ -1,6 +1,7 @@
 require "tzinfo"
 require "perfect_retry"
 require "embulk/input/mixpanel_api/client"
+require "embulk/input/mixpanel_api/exceptions"
 require "range_generator"
 require "timezone_validator"
 
@@ -71,7 +72,8 @@ module Embulk
           incremental: incremental,
           slice_range: config.param(:slice_range, :integer, default: 7),
           job_start_time: job_start_time,
-          incremental_column_upper_limit: incremental_column_upper_limit
+          incremental_column_upper_limit: incremental_column_upper_limit,
+          allow_partial_import: config.param(:allow_partial_import,:bool, default: true)
         }
 
         if !incremental_column.nil? && !latest_fetched_time.nil? && (incremental_column_upper_limit <= latest_fetched_time)
@@ -144,7 +146,7 @@ module Embulk
         PerfectRetry.new do |config|
           config.limit = task[:retry_limit]
           config.sleep = proc{|n| task[:retry_initial_wait_sec] * (2 * (n - 1)) }
-          config.dont_rescues = [Embulk::ConfigError]
+          config.dont_rescues = [Embulk::ConfigError,MixpanelApi::IncompleteExportResponseError]
           config.rescues = [RuntimeError]
           config.log_level = nil
           config.logger = Embulk.logger
@@ -175,33 +177,40 @@ module Embulk
             Embulk.logger.info "Fetching data from #{slice_dates.first} to #{slice_dates.last} ..."
           end
           record_time_column=@incremental_column || DEFAULT_TIME_COLUMN
-          fetch(slice_dates, prev_latest_fetched_time).each do |record|
-            if @incremental
-              if !record["properties"].include?(record_time_column)
-                raise Embulk::ConfigError.new("Incremental column not exists in fetched data #{record_time_column}")
-              end
-              record_time = record["properties"][record_time_column]
-              if @incremental_column.nil?
-                if record_time <= prev_latest_fetched_time
-                  ignored_record_count += 1
-                  next
+          begin
+            fetch(slice_dates, prev_latest_fetched_time).each do |record|
+              if @incremental
+                if !record["properties"].include?(record_time_column)
+                  raise Embulk::ConfigError.new("Incremental column not exists in fetched data #{record_time_column}")
+                end
+                record_time = record["properties"][record_time_column]
+                if @incremental_column.nil?
+                  if record_time <= prev_latest_fetched_time
+                    ignored_record_count += 1
+                    next
+                  end
                 end
-              end
 
-              current_latest_fetched_time= [
-                current_latest_fetched_time,
-                record_time,
-              ].max
-            end
-            values = extract_values(record)
-            if @fetch_unknown_columns
-              unknown_values = extract_unknown_values(record)
-              values << unknown_values.to_json
+                current_latest_fetched_time= [
+                  current_latest_fetched_time,
+                  record_time,
+                ].max
+              end
+              values = extract_values(record)
+              if @fetch_unknown_columns
+                unknown_values = extract_unknown_values(record)
+                values << unknown_values.to_json
+              end
+              if task[:fetch_custom_properties]
+                values << collect_custom_properties(record)
+              end
+              page_builder.add(values)
             end
-            if task[:fetch_custom_properties]
-              values << collect_custom_properties(record)
+          rescue MixpanelApi::IncompleteExportResponseError
+            if !task[:allow_partial_import]
+            #   re raise the exception if we don't allow partial import
+              raise
             end
-            page_builder.add(values)
           end
           if ignored_record_count > 0
             Embulk.logger.warn "Skipped already loaded #{ignored_record_count} records. These record times are older or equal than previous fetched record time (#{prev_latest_fetched_time} @ #{prev_latest_fetched_time_format})."

data/lib/embulk/input/mixpanel_api/client.rb

@@ -2,6 +2,7 @@ require "uri"
 require "digest/md5"
 require "json"
 require "httpclient"
+require "embulk/input/mixpanel_api/exceptions"
 
 module Embulk
   module Input
@@ -118,6 +119,11 @@ module Embulk
             end
           end
           handle_error(response, error_response)
+          if !buf.empty?
+            #   buffer is not empty mean the last json line is incomplete
+            Embulk.logger.error "Received incomplete data from Mixpanel, #{buf}"
+            raise MixpanelApi::IncompleteExportResponseError.new("Incomplete data received")
+          end
         end
 
         def request_small_dataset(params, range)

data/lib/embulk/input/mixpanel_api/exceptions.rb

@@ -0,0 +1,7 @@
+module Embulk
+  module Input
+    module MixpanelApi
+      class IncompleteExportResponseError < StandardError; end
+    end
+  end
+end

data/test/embulk/input/mixpanel_api/test_client.rb

@@ -1,4 +1,5 @@
 require "embulk/input/mixpanel_api/client"
+require "embulk/input/mixpanel_api/exceptions"
 require "override_assert_raise"
 
 module Embulk
@@ -87,6 +88,33 @@ module Embulk
             assert_equal(dummy_responses, records)
           end
 
+          def test_export_partial_with_export_terminated_early
+            stub_client
+            stub(@client).set_signatures(anything) {}
+            stub_response(Struct.new(:code, :body).new(200, jsonl_dummy_responses+"\nexport terminated early"))
+
+            records = []
+            assert_raise MixpanelApi::IncompleteExportResponseError do
+              @client.export(params) do |record|
+                records << record
+              end
+            end
+            assert_equal(dummy_responses, records)
+          end
+
+          def test_export_partial_with_error_json
+            stub_client
+            stub(@client).set_signatures(anything) {}
+            stub_response(Struct.new(:code, :body).new(200, jsonl_dummy_responses+"\n{\"error\":"))
+            records = []
+            assert_raise MixpanelApi::IncompleteExportResponseError do
+              @client.export(params) do |record|
+                records << record
+              end
+            end
+            assert_equal(dummy_responses, records)
+          end
+
           def test_failure_with_400
             stub_client
             stub_response(failure_response(400))

data/test/embulk/input/test_mixpanel.rb

@@ -595,7 +595,32 @@ module Embulk
 
           @plugin.run
         end
+        class PartialRunTest < self
+          def setup_client
+            any_instance_of(MixpanelApi::Client) do |klass|
+              stub(klass).request { raise MixpanelApi::IncompleteExportResponseError.new("Incomplete data received") }
+            end
+          end
+          def setup
+            setup_client()
+            @page_builder = Object.new
+          end
+
+          def test_run_with_allow_partial_false
+            @plugin = Mixpanel.new(task.merge(allow_partial_import: false), nil, nil, @page_builder)
+            stub(@plugin).preview? {false}
+            assert_raise MixpanelApi::IncompleteExportResponseError do
+              @plugin.run
+            end
+          end
 
+          def test_run_with_allow_partial_true
+            @plugin = Mixpanel.new(task.merge(allow_partial_import: true), nil, nil, @page_builder)
+            mock(@page_builder).finish
+            stub(@plugin).preview? {false}
+            @plugin.run
+          end
+        end
         class SliceRangeRunTest < self
 
           def test_default_slice_range
@@ -725,7 +750,6 @@ module Embulk
 
             @plugin.run
           end
-
           private
 
           def task
@@ -834,7 +858,8 @@ module Embulk
           latest_fetched_time: 0,
           slice_range: 7,
           job_start_time: JOB_START_TIME,
-          incremental_column_upper_limit: (JOB_START_TIME - UPPER_LIMIT_DELAY * 1000)
+          incremental_column_upper_limit: (JOB_START_TIME - UPPER_LIMIT_DELAY * 1000),
+          allow_partial_import: true
         }
       end
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: embulk-input-mixpanel
 version: !ruby/object:Gem::Version
-  version: 0.5.9
+  version: 0.5.10
 platform: ruby
 authors:
 - yoshihara
@@ -9,80 +9,79 @@ authors:
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2017-11-10 00:00:00.000000000 Z
+date: 2017-12-03 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  name: httpclient
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: 2.8.3
+  name: httpclient
+  prerelease: false
+  type: :runtime
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: 2.8.3
-  prerelease: false
-  type: :runtime
 - !ruby/object:Gem::Dependency
-  name: tzinfo
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+  name: tzinfo
+  prerelease: false
+  type: :runtime
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-  prerelease: false
-  type: :runtime
 - !ruby/object:Gem::Dependency
-  name: perfect_retry
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '0.5'
+  name: perfect_retry
+  prerelease: false
+  type: :runtime
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '0.5'
-  prerelease: false
-  type: :runtime
 - !ruby/object:Gem::Dependency
-  name: bundler
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '1.0'
+  name: bundler
+  prerelease: false
+  type: :development
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '1.0'
-  prerelease: false
-  type: :development
 - !ruby/object:Gem::Dependency
-  name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: '10.0'
+  name: rake
+  prerelease: false
+  type: :development
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: '10.0'
-  prerelease: false
-  type: :development
 - !ruby/object:Gem::Dependency
-  name: embulk
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -91,6 +90,9 @@ dependencies:
     - - "<"
       - !ruby/object:Gem::Version
         version: '1.0'
+  name: embulk
+  prerelease: false
+  type: :development
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -99,78 +101,76 @@ dependencies:
     - - "<"
       - !ruby/object:Gem::Version
         version: '1.0'
-  prerelease: false
-  type: :development
 - !ruby/object:Gem::Dependency
-  name: pry
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+  name: pry
+  prerelease: false
+  type: :development
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-  prerelease: false
-  type: :development
 - !ruby/object:Gem::Dependency
-  name: test-unit
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+  name: test-unit
+  prerelease: false
+  type: :development
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-  prerelease: false
-  type: :development
 - !ruby/object:Gem::Dependency
-  name: test-unit-rr
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+  name: test-unit-rr
+  prerelease: false
+  type: :development
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-  prerelease: false
-  type: :development
 - !ruby/object:Gem::Dependency
-  name: codeclimate-test-reporter
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '0.5'
+  name: codeclimate-test-reporter
+  prerelease: false
+  type: :development
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '0.5'
-  prerelease: false
-  type: :development
 - !ruby/object:Gem::Dependency
-  name: everyleaf-embulk_helper
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+  name: everyleaf-embulk_helper
+  prerelease: false
+  type: :development
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-  prerelease: false
-  type: :development
 description: Loads records from Mixpanel.
 email:
 - h.yoshihara@everyleaf.com
@@ -195,6 +195,7 @@ files:
 - gemfiles/template.erb
 - lib/embulk/input/mixpanel.rb
 - lib/embulk/input/mixpanel_api/client.rb
+- lib/embulk/input/mixpanel_api/exceptions.rb
 - lib/range_generator.rb
 - lib/timezone_validator.rb
 - test/embulk/input/mixpanel_api/test_client.rb