embulk-input-mixpanel 0.3.4 → 0.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 4f7e9eb914b30c6a19d3afe35b5b5314497c0c83
-  data.tar.gz: 1b3d12b83c9c06d66657b6e370ad2c28446b1754
+  metadata.gz: 0dedffafe59dd4e3dcbe23d13d3d2ae8f1a4046f
+  data.tar.gz: dad21a163d3dfe678e3b4d49531c476fe48af594
 SHA512:
-  metadata.gz: 20a77df82f23049667d97de1a245a46844733983c5fece681dd53857507ec7eae6b1ee5a6ccefe325c48038c215584bf5590c93f710af44a64bfa1c206c4712b
-  data.tar.gz: d9536d638c52cac6af9936bea589b408a8f1106509ed6a60d929c3cec06d2bb96d29444a7666194b50ed581d210df599914f8d979d37a23a7ac0e8cf30b2bfe7
+  metadata.gz: 0ac29f80faece33812fb85d48aed368398ea1fb2ee779cab01b7489ea1f171cfa7ea6aa9d017527d6ab8ca80b6966f29c22164ca0534f3878c1f94e2cd92b56c
+  data.tar.gz: c4a4382fd745a91c483ae5eff4b11e2aca0949a157ddcfb127d2430d273d4158c8087b350d0da5844e957a487a1c08e7bd0b1c07cef30900c23db883e870b566

data/.ruby-version

@@ -1 +1 @@
-jruby-9.0.1.0
+jruby-9.0.5.0

data/.travis.yml

@@ -1,84 +1,35 @@
 language: ruby
 
 jdk: oraclejdk8
+sudo: false
 
 addons:
   code_climate:
     repo_token:
       secure: "opE/ZhRzsEU2Fn6YEnItMD/rMs3O2OHVYQQ7Ly0dAkIc9ZrVMa//ogt4k7h0QGZUettRVV7kawCtRzde+QbOmezTRwWqQJ7Mi6D6qfwlYMz+D9FcufHaCJyLy+dYeBuQqr1d6n/3gVqVGh8MAMGHipYodudCub38DQ1sVWcCMDNNo4PMRFLS0pS839SC80HAS7tutOeaeohRc+Ct5yoY2ZDoTuGtNeJhwqmDAo13RwvVu9aZw97EZPvt8UcaW5oYDnx47kDpKi4XGrTPaWpSm/IitwW11FF+Kevt0RpUS0uVWqg4/6xTjDg++ETG+94ePYrOZGF4ne/CPtC0AtaWF1jgSlL9iu4IR/Awt+2BqawKzCnoSRgCGIHRZApErC3KacadJBaPCrKwf4xxxqRXex4lXcptKRygkG4ic2r+MblyPPwIsc1Wb1QYCeVjIEGWzOgKVEUpT8qN0DAj6KQe/HuyuXKE/FiPIRfkJkRY5oGCNZtzeCzXlC+IqhZYCg6HwcZuk3CbcxRrg5VFKDlL0VWacD/FQoGgfTp3SUmaL6NMcCKSrL0vjEgx98/yM9rsGZlZOU9ioN93PQVgqpI5dl+nm9vKkFyuzW07nNM+/6PNdHbbZBY1OLNd3RSpWVC9TDtZ3Q2gAx25+31TDzD2/3yjJjfdtwIH3bdyrO44MK8="
-notifications:
-  slack:
-    secure: "cAAHcb/HdJK6elr6gfxlotUJdZys8nA/BUb1rKVt7T7EShOjqNOwHo0vZBG5fXQEbUJjHCFAl8tH1N8wz+KLoVErlMmD2OgBKGgvgm3fddkIgGRRvxe8yB0yEFy5yuvfgqJDlwPHZxbwj2Ddb8uTaLd8yzmRzGWNwLyRQTDaQbaKBaB+emP2t/1AilQHyItN02NuW3qcEGr7Hzg+IsKabDRzhMi89RV0xk2eP5iWTYW76ULbbfP3NqBpcVLskAqVQd1YOFUSEb2fdzRj7UuLUUp3C0Kb5gKfXPHb97XuMqwyviakTr02omdaM82NvB8GKn7QKBbcwA1Iz1euinxQQ7MTzJe8wLMH8q205lTpl96CANKwIxrk/HestgEboq97xSp5Wc1Qn1ynnrLOvBYW7EG/e6exguojQmkLkV9P6LW9g4nYsR8UEjyuRd252xch4+b4M3lxtlAXtpWguPfm5ZKhICBsJMlYm6//q1cBlNUOzR0cufCHDTNdlB/YLynvdg1ykFISYARP+5sV2YM9rD/1Rs/396qYr0mNbCh83P9GlZ5NZtWLQePJroHe3iQ18X3YHEcb50OTfRmKNoXTldQBD9iKNkwlFpWfruVdBWT+JRM77+yIV9ORjY7UE4ZtH8HxAg4bX+ey+C5axgxQk569h3CdLmVapFt6wovCSVk="
 
 before_install:
+  - rvm list
   - |
-    # Currently, Travis can't treat jruby 9.0.1.0
-    if [[ ${TRAVIS_RUBY_VERSION} = "jruby-9.0.0.0" ]];then
-      rvm get head
-      rvm use jruby-9.0.1.0 --install
-      ruby -v
-    fi
+    # Currently, Travis can't treat jruby 9.0.5.0
+    ruby -v
+    rvm get head
+    rvm use jruby-9.0.5.0 --install
+    gem install bundler
+    ruby -v
+  - gem i bundler
 
 rvm:
-  - jruby-19mode
-  - jruby-9.0.0.0
+  - jruby-9.0.5.0
 
 gemfile:
-  - gemfiles/embulk-0.6.0-latest
-  - gemfiles/embulk-0.6.16
-  - gemfiles/embulk-0.6.17
-  - gemfiles/embulk-0.6.18
-  - gemfiles/embulk-0.6.19
-  - gemfiles/embulk-0.6.20
-  - gemfiles/embulk-0.6.21
-  - gemfiles/embulk-0.6.22
-  - gemfiles/embulk-0.6.23
-  - gemfiles/embulk-0.6.24
-  - gemfiles/embulk-0.6.25
-  - gemfiles/embulk-0.6.26
-  - gemfiles/embulk-0.6.27
-  - gemfiles/embulk-0.7.0
-  - gemfiles/embulk-0.7.0-latest
-  - gemfiles/embulk-0.7.1
-  - gemfiles/embulk-0.7.2
-  - gemfiles/embulk-0.7.3
-  - gemfiles/embulk-0.7.4
-  - gemfiles/embulk-0.7.5
+  - gemfiles/embulk-0.8.0-latest
+  - gemfiles/embulk-0.8.6
   - gemfiles/embulk-latest
 
 matrix:
-  exclude:
-    - jdk: oraclejdk8 # Ignore all matrix at first, use `include` to allow build
-  include:
-    - {rvm: jruby-19mode, gemfile: gemfiles/embulk-0.6.0-latest}
-    - {rvm: jruby-19mode, gemfile: gemfiles/embulk-0.6.16}
-    - {rvm: jruby-19mode, gemfile: gemfiles/embulk-0.6.17}
-    - {rvm: jruby-19mode, gemfile: gemfiles/embulk-0.6.18}
-    - {rvm: jruby-19mode, gemfile: gemfiles/embulk-0.6.19}
-    - {rvm: jruby-19mode, gemfile: gemfiles/embulk-0.6.20}
-    - {rvm: jruby-19mode, gemfile: gemfiles/embulk-0.6.21}
-    - {rvm: jruby-19mode, gemfile: gemfiles/embulk-0.6.22}
-    - {rvm: jruby-19mode, gemfile: gemfiles/embulk-0.6.23}
-    - {rvm: jruby-19mode, gemfile: gemfiles/embulk-0.6.24}
-    - {rvm: jruby-19mode, gemfile: gemfiles/embulk-0.6.25}
-    - {rvm: jruby-19mode, gemfile: gemfiles/embulk-0.6.26}
-    - {rvm: jruby-19mode, gemfile: gemfiles/embulk-0.6.27}
-    - {rvm: jruby-9.0.0.0, gemfile: gemfiles/embulk-0.7.0}
-    - {rvm: jruby-9.0.0.0, gemfile: gemfiles/embulk-0.7.0-latest}
-    - {rvm: jruby-9.0.0.0, gemfile: gemfiles/embulk-0.7.1}
-    - {rvm: jruby-9.0.0.0, gemfile: gemfiles/embulk-0.7.2}
-    - {rvm: jruby-9.0.0.0, gemfile: gemfiles/embulk-0.7.3}
-    - {rvm: jruby-9.0.0.0, gemfile: gemfiles/embulk-0.7.4}
-    - {rvm: jruby-9.0.0.0, gemfile: gemfiles/embulk-0.7.5}
-    - {rvm: jruby-9.0.0.0, gemfile: gemfiles/embulk-latest}
-    
-
   allow_failures:
-    - gemfile: gemfiles/embulk-0.6.22
-    - gemfile: gemfiles/embulk-0.7.0
-    - gemfile: gemfiles/embulk-0.7.1
     # Ignore failure for *-latest
-    - gemfile: embulk-0.6.0-latest
-    - gemfile: embulk-0.7.0-latest
+    - gemfile: embulk-0.8.0-latest
     - gemfile: embulk-latest
     

data/.travis.yml.erb

@@ -1,27 +1,26 @@
 language: ruby
 
 jdk: oraclejdk8
+sudo: false
 
 addons:
   code_climate:
     repo_token:
       secure: "opE/ZhRzsEU2Fn6YEnItMD/rMs3O2OHVYQQ7Ly0dAkIc9ZrVMa//ogt4k7h0QGZUettRVV7kawCtRzde+QbOmezTRwWqQJ7Mi6D6qfwlYMz+D9FcufHaCJyLy+dYeBuQqr1d6n/3gVqVGh8MAMGHipYodudCub38DQ1sVWcCMDNNo4PMRFLS0pS839SC80HAS7tutOeaeohRc+Ct5yoY2ZDoTuGtNeJhwqmDAo13RwvVu9aZw97EZPvt8UcaW5oYDnx47kDpKi4XGrTPaWpSm/IitwW11FF+Kevt0RpUS0uVWqg4/6xTjDg++ETG+94ePYrOZGF4ne/CPtC0AtaWF1jgSlL9iu4IR/Awt+2BqawKzCnoSRgCGIHRZApErC3KacadJBaPCrKwf4xxxqRXex4lXcptKRygkG4ic2r+MblyPPwIsc1Wb1QYCeVjIEGWzOgKVEUpT8qN0DAj6KQe/HuyuXKE/FiPIRfkJkRY5oGCNZtzeCzXlC+IqhZYCg6HwcZuk3CbcxRrg5VFKDlL0VWacD/FQoGgfTp3SUmaL6NMcCKSrL0vjEgx98/yM9rsGZlZOU9ioN93PQVgqpI5dl+nm9vKkFyuzW07nNM+/6PNdHbbZBY1OLNd3RSpWVC9TDtZ3Q2gAx25+31TDzD2/3yjJjfdtwIH3bdyrO44MK8="
-notifications:
-  slack:
-    secure: "cAAHcb/HdJK6elr6gfxlotUJdZys8nA/BUb1rKVt7T7EShOjqNOwHo0vZBG5fXQEbUJjHCFAl8tH1N8wz+KLoVErlMmD2OgBKGgvgm3fddkIgGRRvxe8yB0yEFy5yuvfgqJDlwPHZxbwj2Ddb8uTaLd8yzmRzGWNwLyRQTDaQbaKBaB+emP2t/1AilQHyItN02NuW3qcEGr7Hzg+IsKabDRzhMi89RV0xk2eP5iWTYW76ULbbfP3NqBpcVLskAqVQd1YOFUSEb2fdzRj7UuLUUp3C0Kb5gKfXPHb97XuMqwyviakTr02omdaM82NvB8GKn7QKBbcwA1Iz1euinxQQ7MTzJe8wLMH8q205lTpl96CANKwIxrk/HestgEboq97xSp5Wc1Qn1ynnrLOvBYW7EG/e6exguojQmkLkV9P6LW9g4nYsR8UEjyuRd252xch4+b4M3lxtlAXtpWguPfm5ZKhICBsJMlYm6//q1cBlNUOzR0cufCHDTNdlB/YLynvdg1ykFISYARP+5sV2YM9rD/1Rs/396qYr0mNbCh83P9GlZ5NZtWLQePJroHe3iQ18X3YHEcb50OTfRmKNoXTldQBD9iKNkwlFpWfruVdBWT+JRM77+yIV9ORjY7UE4ZtH8HxAg4bX+ey+C5axgxQk569h3CdLmVapFt6wovCSVk="
 
 before_install:
+  - rvm list
   - |
-    # Currently, Travis can't treat jruby 9.0.1.0
-    if [[ ${TRAVIS_RUBY_VERSION} = "jruby-9.0.0.0" ]];then
-      rvm get head
-      rvm use jruby-9.0.1.0 --install
-      ruby -v
-    fi
+    # Currently, Travis can't treat jruby 9.0.5.0
+    ruby -v
+    rvm get head
+    rvm use jruby-9.0.5.0 --install
+    gem install bundler
+    ruby -v
+  - gem i bundler
 
 rvm:
-  - jruby-19mode
-  - jruby-9.0.0.0
+  - jruby-9.0.5.0
 
 gemfile:
 <% versions.each do |file| -%>
@@ -29,17 +28,7 @@ gemfile:
 <% end -%>
 
 matrix:
-  exclude:
-    - jdk: oraclejdk8 # Ignore all matrix at first, use `include` to allow build
-  include:
-    <% matrix.each do |m| -%>
-<%= m %>
-    <% end %>
-
   allow_failures:
-    - gemfile: gemfiles/embulk-0.6.22
-    - gemfile: gemfiles/embulk-0.7.0
-    - gemfile: gemfiles/embulk-0.7.1
     # Ignore failure for *-latest
     <% versions.find_all{|file| file.to_s.match(/-latest/)}.each do |file| -%>
 - gemfile: <%= file %>

data/CHANGELOG.md

@@ -1,3 +1,14 @@
+## 0.4.0 - 2016-03-04
+
+This version contains compatibility breaking. Only support Embulk 0.8 or later since this version, no longer support Embulk 0.7.x or earlier.
+
+* [enhancement] Support json type [#35](https://github.com/treasure-data/embulk-input-mixpanel/pull/35)
+* [enhancement] Check Mixpanel availability before run [#37](https://github.com/treasure-data/embulk-input-mixpanel/pull/37)
+* [enhancement] Guessing time column as statically [#36](https://github.com/treasure-data/embulk-input-mixpanel/pull/36)
+* [enhancement] Reduce guess and preview records [#34](https://github.com/treasure-data/embulk-input-mixpanel/pull/34)
+* [maintenance] Use perfect_retry [#33](https://github.com/treasure-data/embulk-input-mixpanel/pull/33)
+
+
 ## 0.3.4 - 2015-11-02
 
 * [enhancement] Create `unknown_columns` only when option is true [#32](https://github.com/treasure-data/embulk-input-mixpanel/pull/32)

data/embulk-input-mixpanel.gemspec

@@ -1,7 +1,7 @@
 
 Gem::Specification.new do |spec|
   spec.name          = "embulk-input-mixpanel"
-  spec.version       = "0.3.4"
+  spec.version       = "0.4.0"
   spec.authors       = ["yoshihara", "uu59"]
   spec.summary       = "Mixpanel input plugin for Embulk"
   spec.description   = "Loads records from Mixpanel."
@@ -15,9 +15,10 @@ Gem::Specification.new do |spec|
 
   spec.add_dependency 'httpclient'
   spec.add_dependency 'tzinfo'
+  spec.add_dependency 'perfect_retry', ["~> 0.3"]
   spec.add_development_dependency 'bundler', ['~> 1.0']
   spec.add_development_dependency 'rake', ['>= 10.0']
-  spec.add_development_dependency 'embulk', ['>= 0.6.16', '< 1.0']
+  spec.add_development_dependency 'embulk', ['>= 0.8.6', '< 1.0']
   spec.add_development_dependency 'pry'
   spec.add_development_dependency 'test-unit'
   spec.add_development_dependency 'test-unit-rr'

data/gemfiles/{embulk-0.6.0-latest → embulk-0.8.0-latest}

@@ -1,4 +1,4 @@
 source 'https://rubygems.org/'
 gemspec :path => '../'
 
-gem "embulk", "~> 0.6.0"
+gem "embulk", "~> 0.8.0"

data/gemfiles/{embulk-0.7.0 → embulk-0.8.6}

@@ -1,4 +1,4 @@
 source 'https://rubygems.org/'
 gemspec :path => '../'
 
-gem "embulk", "0.7.0"
+gem "embulk", "0.8.6"

data/gemfiles/embulk-latest

@@ -1,4 +1,4 @@
 source 'https://rubygems.org/'
 gemspec :path => '../'
 
-gem "embulk", "> 0.6.16"
+gem "embulk", ">= 0.8.6"

data/lib/embulk/input/mixpanel.rb

@@ -1,4 +1,5 @@
 require "tzinfo"
+require "perfect_retry"
 require "embulk/input/mixpanel_api/client"
 require "range_generator"
 require "timezone_validator"
@@ -47,7 +48,7 @@ module Embulk
         end
 
         if fetch_unknown_columns
-          columns << Column.new(nil, "unknown_columns", :string)
+          columns << Column.new(nil, "unknown_columns", :json)
         end
 
         resume(task, columns, 1, &control)
@@ -66,17 +67,19 @@ module Embulk
       end
 
       def self.guess(config)
+        giveup_when_mixpanel_is_down
+
         client = MixpanelApi::Client.new(config.param(:api_key, :string), config.param(:api_secret, :string))
 
         range = guess_range(config)
         Embulk.logger.info "Guessing schema using #{range.first}..#{range.last} records"
 
         params = export_params(config).merge(
-          from_date: range.first,
-          to_date: range.last,
+          "from_date" => range.first,
+          "to_date" => range.last,
         )
 
-        columns = guess_from_records(client.export(params))
+        columns = guess_from_records(client.export_for_small_dataset(params))
         return {"columns" => columns}
       end
 
@@ -88,11 +91,23 @@ module Embulk
         @schema = task[:schema]
         @dates = task[:dates]
         @fetch_unknown_columns = task[:fetch_unknown_columns]
+        @retryer = PerfectRetry.new do |config|
+          config.limit = task[:retry_limit]
+          config.sleep = proc{|n| task[:retry_initial_wait_sec] * (2 * (n - 1)) }
+          config.dont_rescues = [Embulk::ConfigError]
+          config.rescues = [RuntimeError]
+          config.log_level = nil
+          config.logger = Embulk.logger
+        end
       end
 
       def run
+        self.class.giveup_when_mixpanel_is_down
+
         @dates.each_slice(SLICE_DAYS_COUNT) do |dates|
-          Embulk.logger.info "Fetching data from #{dates.first} to #{dates.last} ..."
+          unless preview?
+            Embulk.logger.info "Fetching data from #{dates.first} to #{dates.last} ..."
+          end
 
           fetch(dates).each do |record|
             values = extract_values(record)
@@ -114,6 +129,12 @@ module Embulk
 
       private
 
+      def self.giveup_when_mixpanel_is_down
+        unless MixpanelApi::Client.mixpanel_available?
+          raise Embulk::DataError.new("Mixpanel service is down. Please retry later.")
+        end
+      end
+
       def extract_values(record)
         @schema.map do |column|
           extract_value(record, column["name"])
@@ -155,7 +176,14 @@ module Embulk
           "to_date" => to_date,
         )
         client = MixpanelApi::Client.new(@api_key, @api_secret)
-        client.export_with_retry(params, task[:retry_initial_wait_sec], task[:retry_limit])
+
+        @retryer.with_retry do
+          if preview?
+            client.export_for_small_dataset(params)
+          else
+            client.export(params)
+          end
+        end
       end
 
       def adjust_timezone(epoch)
@@ -204,14 +232,16 @@ module Embulk
         sample_props = records.first(GUESS_RECORDS_COUNT).map{|r| r["properties"]}
         schema = Guess::SchemaGuess.from_hash_records(sample_props)
         columns = schema.map do |col|
+          next if col.name == "time"
           result = {
             name: col.name,
             type: col.type,
           }
           result[:format] = col.format if col.format
           result
-        end
+        end.compact
         columns.unshift(name: NOT_PROPERTY_COLUMN, type: :string)
+        columns.unshift(name: "time", type: :long)
       end
     end
 

data/lib/embulk/input/mixpanel_api/client.rb

@@ -9,23 +9,55 @@ module Embulk
       class Client
         ENDPOINT_EXPORT = "https://data.mixpanel.com/api/2.0/export/".freeze
         TIMEOUT_SECONDS = 3600
+        PING_TIMEOUT_SECONDS = 3
+        PING_RETRY_LIMIT = 3
+        PING_RETRY_WAIT = 2
+
+        def self.mixpanel_available?
+          retryer = PerfectRetry.new do |config|
+            config.limit = PING_RETRY_LIMIT
+            config.sleep = PING_RETRY_WAIT
+            config.logger = Embulk.logger
+            config.log_level = nil
+          end
+
+          begin
+            retryer.with_retry do
+              client = HTTPClient.new
+              client.connect_timeout = PING_TIMEOUT_SECONDS
+              client.get("https://data.mixpanel.com")
+            end
+            true
+          rescue PerfectRetry::TooManyRetry
+            false
+          end
+        end
 
         def initialize(api_key, api_secret)
           @api_key = api_key
           @api_secret = api_secret
         end
 
-        def export_with_retry(params = {}, retry_initial_wait_sec, retry_limit)
-          body = with_retry(retry_initial_wait_sec, retry_limit) do
-            request(params)
-          end
-
+        def export(params = {})
+          body = request(params)
           response_to_enum(body)
         end
 
-        def export(params = {})
+        def export_for_small_dataset(params = {}, times = 0)
+          days = (1 * (10 ** times))
+          to_date = Date.parse(params["from_date"].to_s) + days
+          params["to_date"] = to_date.strftime("%Y-%m-%d")
+
           body = request(params)
-          response_to_enum(body)
+          result = response_to_enum(body)
+          if result.to_a.length.zero?
+            if times >= 5
+              raise ConfigError.new "#{params["from_date"]} + #{days} days has no record. too old date?"
+            end
+            export_for_small_dataset(params, times + 1)
+          else
+            result
+          end
         end
 
         private
@@ -56,29 +88,9 @@ module Embulk
           response.body
         end
 
-        def with_retry(initial_wait, retry_limit, &block)
-          retry_count = 0
-          wait_sec = initial_wait
-          begin
-            yield
-          rescue Embulk::ConfigError => e # TODO: rescue Embulk::DataError for Embulk 0.7+
-            # Don't retry
-            raise e
-          rescue => e
-            if retry_limit <= retry_count
-              Embulk.logger.error "'#{e}(#{e.class})' error occured and reached retry limit (#{retry_limit} times)"
-              raise e
-            end
-            retry_count += 1
-            Embulk.logger.warn "Retrying after #{wait_sec} seconds [#{retry_count}/#{retry_limit}] '#{e}(#{e.class})' error occured"
-            sleep wait_sec
-            wait_sec *= 2
-            retry
-          end
-        end
-
         def signature(params)
           # https://mixpanel.com/docs/api-documentation/data-export-api#auth-implementation
+          params.delete(:sig)
           sorted_keys = params.keys.map(&:to_s).sort
           signature = sorted_keys.inject("") do |sig, key|
             value = params[key] || params[key.to_sym]

data/test/embulk/input/mixpanel_api/test_client.rb

@@ -89,69 +89,30 @@ module Embulk
             end
           end
 
-          class ExportRetryTest < self
-            def setup
-              @httpclient = HTTPClient.new
-              @client = Client.new(API_KEY, API_SECRET)
-              @retry_initial_wait_sec = 1
-              @retry_limit = 3
-              stub_client
-            end
-
-            def test_retry_with_500
-              stub_response(failure_response(500))
-
-              @retry_limit.times do |n|
-                mock(@client).sleep(@retry_initial_wait_sec * (2**n))
-              end
-              mock(Embulk.logger).warn(/retry/i).times(@retry_limit)
-              mock(Embulk.logger).error(/retry/i).once
+          class ExportSmallDataset < self
+            def test_to_date_after_1_day
+              to = (Date.parse(params["from_date"]) + 1).to_s
+              mock(@client).request(params.merge("to_date" => to)) { jsonl_dummy_responses }
 
-              assert_raise do
-                @client.export_with_retry(params, @retry_initial_wait_sec, @retry_limit)
-              end
+              @client.export_for_small_dataset(params)
             end
 
-            def test_retry_with_timeout
-              @httpclient.connect_timeout = 0.000000000000000000001
-
-              @retry_limit.times do |n|
-                mock(@client).sleep(@retry_initial_wait_sec * (2**n))
-              end
-              mock(Embulk.logger).warn(/retry/i).times(@retry_limit)
-              mock(Embulk.logger).error(/retry/i).once
+            def test_to_date_after_1_day_after_10_days_if_empty
+              to1 = (Date.parse(params["from_date"]) + 1).to_s
+              to2 = (Date.parse(params["from_date"]) + 10).to_s
+              mock(@client).request(params.merge("to_date" => to1)) { "" }
+              mock(@client).request(params.merge("to_date" => to2)) { jsonl_dummy_responses }
 
-              assert_raise(HTTPClient::TimeoutError) do
-                @client.export_with_retry(params, @retry_initial_wait_sec, @retry_limit)
-              end
+              @client.export_for_small_dataset(params)
             end
 
-            def test_not_retry_with_401
-              @httpclient.test_loopback_http_response << "HTTP/1.1 401\r\n\r\n"
-              mock(Embulk.logger).warn(/retry/i).never
-              mock(Embulk.logger).error(/retry/i).never
+            def test_config_error_when_too_long_empty_dates
+              stub(@client).request(anything) { "" }
 
               assert_raise(Embulk::ConfigError) do
-                @client.export_with_retry(params, 0, 1).each do |record|
-                  record
-                end
+                @client.export_for_small_dataset(params)
               end
             end
-
-            def test_not_retry_with_invalid_json
-              omit "Embulk 0.6 or earlier has no DataError (enable to uncomment below code)"
-
-              # @httpclient.test_loopback_http_response << "HTTP/1.1 200\r\n\r\ninvalid json"
-              # mock(Embulk.logger).warn(/retry/i).never
-              # mock(Embulk.logger).error(/retry/i).never
-
-              # assert_raise(Embulk::DataError) do
-              #   @client.export_with_retry(params, 0, 1).each do |record|
-              #     # DataError will raised in each block
-              #     record
-              #   end
-              # end
-            end
           end
 
           private
@@ -176,10 +137,10 @@ module Embulk
 
           def params
             {
-              api_key: API_KEY,
-              api_secret: API_SECRET,
-              from_date: "2015-01-01",
-              to_date: "2015-03-02",
+              "api_key" => API_KEY,
+              "api_secret" => API_SECRET,
+              "from_date" => "2015-01-01",
+              "to_date" => "2015-03-02",
             }
           end
 

data/test/embulk/input/test_mixpanel.rb

@@ -24,6 +24,7 @@ module Embulk
       def setup
         setup_client
         setup_logger
+        stub(Embulk::Input::MixpanelApi::Client).mixpanel_available? { true }
       end
 
       def setup_client
@@ -53,6 +54,7 @@ module Embulk
         def setup
           # Do nothing from parent
           mute_warn
+          stub(Embulk::Input::MixpanelApi::Client).mixpanel_available? { true }
         end
 
         def test_from_date_old_date
@@ -112,11 +114,33 @@ module Embulk
           Mixpanel.guess(embulk_config(config))
         end
 
+        def test_json_type
+          sample_records = records.map do |r|
+            r.merge("properties" => {"array" => [1,2], "hash" => {foo: "FOO"}})
+          end
+          actual = Mixpanel.guess_from_records(sample_records)
+          assert actual.include?(name: "array", type: :json)
+          assert actual.include?(name: "hash", type: :json)
+        end
+
+        def test_mixpanel_is_down
+          stub(Embulk::Input::MixpanelApi::Client).mixpanel_available? { false }
+          config = {
+            type: "mixpanel",
+            api_key: API_KEY,
+            api_secret: API_SECRET,
+          }
+
+          assert_raise(Embulk::DataError) do
+            Mixpanel.guess(embulk_config(config))
+          end
+        end
+
         private
 
         def stub_export_all
           any_instance_of(MixpanelApi::Client) do |klass|
-            stub(klass).export(anything) { records }
+            stub(klass).export_for_small_dataset(anything) { records }
           end
         end
 
@@ -131,9 +155,9 @@ module Embulk
         def expected
           {
             "columns" => [
+              {name: "time", type: :long},
               {name: "event", type: :string},
               {name: "foo", type: :string},
-              {name: "time", type: :long},
               {name: "int", type: :long},
             ]
           }
@@ -364,6 +388,87 @@ module Embulk
         assert_equal(expected, actual)
       end
 
+      sub_test_case "retry" do
+        def setup
+          @page_builder = Object.new
+          @plugin = Mixpanel.new(task, nil, nil, @page_builder)
+          @plugin.init
+          @httpclient = HTTPClient.new
+          stub(HTTPClient).new { @httpclient }
+          stub(@page_builder).add {}
+          stub(@page_builder).finish {}
+          stub(Embulk.logger).warn {}
+          stub(Embulk.logger).info {}
+          stub(Embulk::Input::MixpanelApi::Client).mixpanel_available? { true }
+        end
+
+        test "200" do
+          stub_response(200)
+          mock(Embulk.logger).warn(/Retrying/).never
+          mock(@page_builder).finish
+          @plugin.run
+        end
+
+        test "400" do
+          stub_response(400)
+          mock(Embulk.logger).warn(/Retrying/).never
+          assert_raise(Embulk::ConfigError) do
+            @plugin.run
+          end
+        end
+
+        test "401" do
+          stub_response(401)
+          mock(Embulk.logger).warn(/Retrying/).never
+          assert_raise(Embulk::ConfigError) do
+            @plugin.run
+          end
+        end
+
+        test "500" do
+          stub_response(500)
+          mock(Embulk.logger).warn(/Retrying/).times(task[:retry_limit])
+          assert_raise(PerfectRetry::TooManyRetry) do
+            @plugin.run
+          end
+        end
+
+        test "timeout" do
+          stub(@httpclient).get { raise HTTPClient::TimeoutError, "timeout" }
+          mock(Embulk.logger).warn(/Retrying/).times(task[:retry_limit])
+
+          assert_raise(PerfectRetry::TooManyRetry) do
+            @plugin.run
+          end
+        end
+
+        test "Mixpanel is down" do
+          stub(Embulk::Input::MixpanelApi::Client).mixpanel_available? { false }
+
+          assert_raise(Embulk::DataError) do
+            @plugin.run
+          end
+        end
+
+        def stub_response(code)
+          stub(@httpclient.test_loopback_http_response).shift { "HTTP/1.1 #{code}\r\n\r\n" }
+        end
+
+        def task
+          {
+            api_key: API_KEY,
+            api_secret: API_SECRET,
+            timezone: TIMEZONE,
+            schema: schema,
+            dates: DATES.to_a.map(&:to_s),
+            params: Mixpanel.export_params(embulk_config),
+            fetch_unknown_columns: false,
+            retry_initial_wait_sec: 0,
+            retry_limit: 3,
+          }
+        end
+      end
+
       class RunTest < self
         def setup_client
 
@@ -379,14 +484,6 @@ module Embulk
           @plugin = Mixpanel.new(task, nil, nil, @page_builder)
         end
 
-        def test_preview_check
-          mock(@plugin).preview? { true }
-          stub(@page_builder).add(anything)
-          stub(@page_builder).finish
-
-          @plugin.run
-        end
-
         def test_preview
           stub(@plugin).preview? { true }
           mock(@page_builder).add(anything).times(records.length)

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: embulk-input-mixpanel
 version: !ruby/object:Gem::Version
-  version: 0.3.4
+  version: 0.4.0
 platform: ruby
 authors:
 - yoshihara
@@ -9,7 +9,7 @@ authors:
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2015-11-02 00:00:00.000000000 Z
+date: 2016-03-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -39,6 +39,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.3'
+  name: perfect_retry
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.3'
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
@@ -72,7 +86,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.6.16
+        version: 0.8.6
     - - "<"
       - !ruby/object:Gem::Version
         version: '1.0'
@@ -83,7 +97,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.6.16
+        version: 0.8.6
     - - "<"
       - !ruby/object:Gem::Version
         version: '1.0'
@@ -175,26 +189,8 @@ files:
 - README.md
 - Rakefile
 - embulk-input-mixpanel.gemspec
-- gemfiles/embulk-0.6.0-latest
-- gemfiles/embulk-0.6.16
-- gemfiles/embulk-0.6.17
-- gemfiles/embulk-0.6.18
-- gemfiles/embulk-0.6.19
-- gemfiles/embulk-0.6.20
-- gemfiles/embulk-0.6.21
-- gemfiles/embulk-0.6.22
-- gemfiles/embulk-0.6.23
-- gemfiles/embulk-0.6.24
-- gemfiles/embulk-0.6.25
-- gemfiles/embulk-0.6.26
-- gemfiles/embulk-0.6.27
-- gemfiles/embulk-0.7.0
-- gemfiles/embulk-0.7.0-latest
-- gemfiles/embulk-0.7.1
-- gemfiles/embulk-0.7.2
-- gemfiles/embulk-0.7.3
-- gemfiles/embulk-0.7.4
-- gemfiles/embulk-0.7.5
+- gemfiles/embulk-0.8.0-latest
+- gemfiles/embulk-0.8.6
 - gemfiles/embulk-latest
 - gemfiles/template.erb
 - lib/embulk/input/mixpanel.rb

data/gemfiles/embulk-0.6.16

@@ -1,4 +0,0 @@
-source 'https://rubygems.org/'
-gemspec :path => '../'
-
-gem "embulk", "0.6.16"

data/gemfiles/embulk-0.6.17

@@ -1,4 +0,0 @@
-source 'https://rubygems.org/'
-gemspec :path => '../'
-
-gem "embulk", "0.6.17"

data/gemfiles/embulk-0.6.18

@@ -1,4 +0,0 @@
-source 'https://rubygems.org/'
-gemspec :path => '../'
-
-gem "embulk", "0.6.18"

data/gemfiles/embulk-0.6.19

@@ -1,4 +0,0 @@
-source 'https://rubygems.org/'
-gemspec :path => '../'
-
-gem "embulk", "0.6.19"

data/gemfiles/embulk-0.6.20

@@ -1,4 +0,0 @@
-source 'https://rubygems.org/'
-gemspec :path => '../'
-
-gem "embulk", "0.6.20"

data/gemfiles/embulk-0.6.21

@@ -1,4 +0,0 @@
-source 'https://rubygems.org/'
-gemspec :path => '../'
-
-gem "embulk", "0.6.21"

data/gemfiles/embulk-0.6.22

@@ -1,4 +0,0 @@
-source 'https://rubygems.org/'
-gemspec :path => '../'
-
-gem "embulk", "0.6.22"

data/gemfiles/embulk-0.6.23

@@ -1,4 +0,0 @@
-source 'https://rubygems.org/'
-gemspec :path => '../'
-
-gem "embulk", "0.6.23"

data/gemfiles/embulk-0.6.24

@@ -1,4 +0,0 @@
-source 'https://rubygems.org/'
-gemspec :path => '../'
-
-gem "embulk", "0.6.24"

data/gemfiles/embulk-0.6.25

@@ -1,4 +0,0 @@
-source 'https://rubygems.org/'
-gemspec :path => '../'
-
-gem "embulk", "0.6.25"

data/gemfiles/embulk-0.6.26

@@ -1,4 +0,0 @@
-source 'https://rubygems.org/'
-gemspec :path => '../'
-
-gem "embulk", "0.6.26"

data/gemfiles/embulk-0.6.27

@@ -1,4 +0,0 @@
-source 'https://rubygems.org/'
-gemspec :path => '../'
-
-gem "embulk", "0.6.27"

data/gemfiles/embulk-0.7.0-latest

@@ -1,4 +0,0 @@
-source 'https://rubygems.org/'
-gemspec :path => '../'
-
-gem "embulk", "~> 0.7.0"

data/gemfiles/embulk-0.7.1

@@ -1,4 +0,0 @@
-source 'https://rubygems.org/'
-gemspec :path => '../'
-
-gem "embulk", "0.7.1"

data/gemfiles/embulk-0.7.2

@@ -1,4 +0,0 @@
-source 'https://rubygems.org/'
-gemspec :path => '../'
-
-gem "embulk", "0.7.2"

data/gemfiles/embulk-0.7.3

@@ -1,4 +0,0 @@
-source 'https://rubygems.org/'
-gemspec :path => '../'
-
-gem "embulk", "0.7.3"

data/gemfiles/embulk-0.7.4

@@ -1,4 +0,0 @@
-source 'https://rubygems.org/'
-gemspec :path => '../'
-
-gem "embulk", "0.7.4"

data/gemfiles/embulk-0.7.5

@@ -1,4 +0,0 @@
-source 'https://rubygems.org/'
-gemspec :path => '../'
-
-gem "embulk", "0.7.5"