embulk-input-mixpanel 0.5.10 → 0.5.11.alpha

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 2db94c13e223960ee53abf712889a290728870d4
-  data.tar.gz: bae22b3a844dd7b9a3e0aa61cbb7c2c06846137f
+  metadata.gz: 0c8f13b72e590889f06eb6f6db27cb0edb213853
+  data.tar.gz: c9406f58714adadca97e6d395050280bfb81ef35
 SHA512:
-  metadata.gz: 72410d007e0d2035ecbeb1827d226c127fe62434f0e94cf3aa683ca4c1c6ba13c9107a1fdbc03d74dae00da7356034787f3da32417539a26efe0cf42f845278f
-  data.tar.gz: 65c4d423bd208e80a9d72f17d6289cf2369c7b740ecbd7397fafa219672b48ff7bdd59a58953051a4919645bc8f2963c64445edad25ea29eb8b2e490130f491f
+  metadata.gz: 462b652b42397941d65ea31923e03d6f44d51610dbfaaa1159ee8cf793060dc84e08a7b91666516757e134368580949383a2ea89415c2149749c8377fbaba73c
+  data.tar.gz: c9d5a3b5c4d9753cb51421059577743dd8e5a9be41e9a142e647028c635621c6cf2ca43a03873aa24c90125b8972fc3bec09b6b8a7e56070e02c5c546582619c

data/README.md

@@ -33,6 +33,7 @@ To get it, you should log in mixpanel website, and click gear icon at the lower
 
 - **api_key**: project API Key (string, required)
 - **api_secret**: project API Secret (string, required)
+- **export_endpoint**: the Data Export API's endpoint (string, default to "http://data.mixpanel.com/api/2.0/export")
 - **timezone**: project timezone(string, required)
 - **from_date**: From date to export (string, optional, default: today - 2)
   - NOTE: Mixpanel API supports to export data from at least 2 days before to at most the previous day.

data/embulk-input-mixpanel.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |spec|
   spec.name          = "embulk-input-mixpanel"
-  spec.version       = "0.5.10"
+  spec.version       = "0.5.11.alpha"
   spec.authors       = ["yoshihara", "uu59"]
   spec.summary       = "Mixpanel input plugin for Embulk"
   spec.description   = "Loads records from Mixpanel."

data/lib/embulk/input/mixpanel.rb

@@ -60,6 +60,7 @@ module Embulk
           params: export_params(config),
           dates: range,
           timezone: timezone,
+          export_endpoint: export_endpoint(config),
           api_key: config.param(:api_key, :string),
           api_secret: config.param(:api_secret, :string),
           schema: config.param(:columns, :array),
@@ -122,14 +123,16 @@ module Embulk
       end
 
       def self.guess(config)
-        giveup_when_mixpanel_is_down
+        giveup_when_mixpanel_is_down(export_endpoint(config))
 
         retryer = perfect_retry({
           retry_initial_wait_sec: config.param(:retry_initial_wait_sec, :integer, default: 1),
           retry_limit: config.param(:retry_limit, :integer, default: 5),
         })
-
-        client = MixpanelApi::Client.new(config.param(:api_key, :string), config.param(:api_secret, :string), retryer)
+        client = MixpanelApi::Client.new(config.param(:api_key, :string),
+                                         config.param(:api_secret, :string),
+                                         retryer,
+                                         export_endpoint(config))
 
         range = guess_range(config)
         Embulk.logger.info "Guessing schema using #{range.first}..#{range.last} records"
@@ -153,7 +156,12 @@ module Embulk
         end
       end
 
+      def self.export_endpoint(config)
+        config.param(:export_endpoint, :string, default: Embulk::Input::MixpanelApi::Client::DEFAULT_EXPORT_ENDPOINT)
+      end
+
       def init
+        @export_endpoint = task[:export_endpoint]
         @api_key = task[:api_key]
         @api_secret = task[:api_secret]
         @params = task[:params]
@@ -167,7 +175,7 @@ module Embulk
 
       def run
         Embulk.logger.info "Job start time is #{task[:job_start_time]}"
-        self.class.giveup_when_mixpanel_is_down
+        self.class.giveup_when_mixpanel_is_down(task[:export_endpoint])
         prev_latest_fetched_time = task[:latest_fetched_time] || 0
         prev_latest_fetched_time_format = Time.at(prev_latest_fetched_time).strftime("%F %T %z")
         current_latest_fetched_time = prev_latest_fetched_time
@@ -227,8 +235,8 @@ module Embulk
 
       private
 
-      def self.giveup_when_mixpanel_is_down
-        unless MixpanelApi::Client.mixpanel_available?
+      def self.giveup_when_mixpanel_is_down(export_endpoint)
+        unless MixpanelApi::Client.mixpanel_available?(export_endpoint)
           raise Embulk::DataError.new("Mixpanel service is down. Please retry later.")
         end
       end
@@ -289,7 +297,7 @@ module Embulk
           )
         end
         Embulk.logger.info "Where params is #{params["where"]}"
-        client = MixpanelApi::Client.new(@api_key, @api_secret, self.class.perfect_retry(task))
+        client = MixpanelApi::Client.new(@api_key, @api_secret, self.class.perfect_retry(task), @export_endpoint)
 
         if preview?
           client.export_for_small_dataset(params)

data/lib/embulk/input/mixpanel_api/client.rb

@@ -8,16 +8,17 @@ module Embulk
   module Input
     module MixpanelApi
       class Client
-        ENDPOINT_EXPORT = "https://data.mixpanel.com/api/2.0/export/".freeze
         TIMEOUT_SECONDS = 3600
         PING_TIMEOUT_SECONDS = 3
         PING_RETRY_LIMIT = 3
         PING_RETRY_WAIT = 2
         SMALLSET_BYTE_RANGE = "0-#{5 * 1024 * 1024}"
+        DEFAULT_EXPORT_ENDPOINT = "https://data.mixpanel.com/api/2.0/export/".freeze
 
         attr_reader :retryer
 
-        def self.mixpanel_available?
+        def self.mixpanel_available?(endpoint = nil)
+          endpoint ||= DEFAULT_EXPORT_ENDPOINT
           retryer = PerfectRetry.new do |config|
             config.limit = PING_RETRY_LIMIT
             config.sleep = PING_RETRY_WAIT
@@ -29,7 +30,7 @@ module Embulk
             retryer.with_retry do
               client = HTTPClient.new
               client.connect_timeout = PING_TIMEOUT_SECONDS
-              client.get("https://data.mixpanel.com")
+              client.get(URI.join(endpoint, '/'))
             end
             true
           rescue PerfectRetry::TooManyRetry
@@ -37,7 +38,8 @@ module Embulk
           end
         end
 
-        def initialize(api_key, api_secret, retryer = nil)
+        def initialize(api_key, api_secret, retryer = nil, endpoint = DEFAULT_EXPORT_ENDPOINT)
+          @endpoint = endpoint
           @api_key = api_key
           @api_secret = api_secret
           @retryer = retryer || PerfectRetry.new do |config|
@@ -102,7 +104,8 @@ module Embulk
 
           buf = ""
           error_response = ''
-          response = httpclient.get(ENDPOINT_EXPORT, params) do |response, chunk|
+          Embulk.logger.info "Sending request to #{@endpoint}"
+          response = httpclient.get(@endpoint, params) do |response, chunk|
             # Only process data if response status is 200..299
             if response.status/100 == 2
               chunk.each_line do |line|
@@ -130,10 +133,11 @@ module Embulk
           # guess/preview
           # Try to fetch first `range` bytes
           set_signatures(params)
-          res = httpclient.get(ENDPOINT_EXPORT, params, {"Range" => "bytes=#{range}"})
+          Embulk.logger.info "Sending request to #{@endpoint}"
+          res = httpclient.get(@endpoint, params, {"Range" => "bytes=#{range}"})
           if res.code == 416
             # cannot satisfied requested Range, get full body
-            res = httpclient.get(ENDPOINT_EXPORT, params)
+            res = httpclient.get(@endpoint, params)
           end
           handle_error(res,res.body)
           response_to_enum(res.body)

data/test/embulk/input/test_mixpanel.rb

@@ -541,6 +541,7 @@ module Embulk
           {
             api_key: API_KEY,
             api_secret: API_SECRET,
+            export_endpoint: "https://data.mixpanel.com/api/2.0/export/",
             timezone: TIMEZONE,
             schema: schema,
             dates: DATES.to_a.map(&:to_s),
@@ -845,6 +846,7 @@ module Embulk
         {
           api_key: API_KEY,
           api_secret: API_SECRET,
+          export_endpoint: "https://data.mixpanel.com/api/2.0/export/",
           timezone: TIMEZONE,
           incremental: true,
           incremental_column: nil,

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: embulk-input-mixpanel
 version: !ruby/object:Gem::Version
-  version: 0.5.10
+  version: 0.5.11.alpha
 platform: ruby
 authors:
 - yoshihara
@@ -9,7 +9,7 @@ authors:
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2017-12-03 00:00:00.000000000 Z
+date: 2017-12-11 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -220,12 +220,12 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">="
+  - - ">"
     - !ruby/object:Gem::Version
-      version: '0'
+      version: 1.3.1
 requirements: []
 rubyforge_project:
-rubygems_version: 2.6.6
+rubygems_version: 2.4.8
 signing_key:
 specification_version: 4
 summary: Mixpanel input plugin for Embulk