embulk-input-mixpanel 0.4.0 → 0.4.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 0dedffafe59dd4e3dcbe23d13d3d2ae8f1a4046f
-  data.tar.gz: dad21a163d3dfe678e3b4d49531c476fe48af594
+  metadata.gz: 7253da23180b92cc1403e3ee23b526d154a8e243
+  data.tar.gz: c1e09f40c320c7959a86db23d4ba1d5562167597
 SHA512:
-  metadata.gz: 0ac29f80faece33812fb85d48aed368398ea1fb2ee779cab01b7489ea1f171cfa7ea6aa9d017527d6ab8ca80b6966f29c22164ca0534f3878c1f94e2cd92b56c
-  data.tar.gz: c4a4382fd745a91c483ae5eff4b11e2aca0949a157ddcfb127d2430d273d4158c8087b350d0da5844e957a487a1c08e7bd0b1c07cef30900c23db883e870b566
+  metadata.gz: bac8feab3089bdd03b8df1758db23a153f3a39edba8e8d10dde9b6b7e9224271985466a8fc8114b308e7b6994360d43b9abf6a1b9246a462c74d3ef562ebdd63
+  data.tar.gz: 44d4a809616745c4ee595a2eb476df8897936d2531e1d847414b26e1211b8dcfdedcb1da95e49fb908ecf059b103670425ef0c3899614d5443ba10ea4fe38468

data/CHANGELOG.md

@@ -1,3 +1,6 @@
+## 0.4.1 - 2016-03-08
+* [enhancement] Reduce data bytes with range [#38](https://github.com/treasure-data/embulk-input-mixpanel/pull/38)
+
 ## 0.4.0 - 2016-03-04
 
 This version contains compatibility breaking. Only support Embulk 0.8 or later since this version, no longer support Embulk 0.7.x or earlier.

data/README.md

@@ -8,7 +8,7 @@ embulk-input-mixpanel is the Embulk input plugin for [Mixpanel](https://mixpanel
 
 ## Overview
 
-Required Embulk version >= 0.6.16.
+Required Embulk version >= 0.8.6 (since v0.4.0).
 
 * **Plugin type**: input
 * **Resume supported**: no

data/embulk-input-mixpanel.gemspec

@@ -1,7 +1,7 @@
 
 Gem::Specification.new do |spec|
   spec.name          = "embulk-input-mixpanel"
-  spec.version       = "0.4.0"
+  spec.version       = "0.4.1"
   spec.authors       = ["yoshihara", "uu59"]
   spec.summary       = "Mixpanel input plugin for Embulk"
   spec.description   = "Loads records from Mixpanel."

data/lib/embulk/input/mixpanel_api/client.rb

@@ -12,6 +12,7 @@ module Embulk
         PING_TIMEOUT_SECONDS = 3
         PING_RETRY_LIMIT = 3
         PING_RETRY_WAIT = 2
+        SMALLSET_BYTE_RANGE = "0-#{5 * 1024 * 1024}"
 
         def self.mixpanel_available?
           retryer = PerfectRetry.new do |config|
@@ -48,9 +49,9 @@ module Embulk
           to_date = Date.parse(params["from_date"].to_s) + days
           params["to_date"] = to_date.strftime("%Y-%m-%d")
 
-          body = request(params)
+          body = request(params, SMALLSET_BYTE_RANGE)
           result = response_to_enum(body)
-          if result.to_a.length.zero?
+          if result.first.nil?
             if times >= 5
               raise ConfigError.new "#{params["from_date"]} + #{days} days has no record. too old date?"
             end
@@ -71,13 +72,20 @@ module Embulk
           end
         end
 
-        def request(params)
+        def request(params, range = nil)
           # https://mixpanel.com/docs/api-documentation/exporting-raw-data-you-inserted-into-mixpanel
           params[:expire] ||= Time.now.to_i + TIMEOUT_SECONDS
           params[:sig] = signature(params)
           Embulk.logger.debug "Export param: #{params.to_s}"
 
-          response = httpclient.get(ENDPOINT_EXPORT, params)
+          headers = {}
+          response =
+            if range
+              # guess/preview
+              httpclient.get(ENDPOINT_EXPORT, params, {"Range" => "bytes=#{range}"})
+            else
+              httpclient.get(ENDPOINT_EXPORT, params)
+            end
           Embulk.logger.debug "response code: #{response.code}"
           case response.code
           when 400..499

data/test/embulk/input/mixpanel_api/test_client.rb

@@ -92,7 +92,7 @@ module Embulk
           class ExportSmallDataset < self
             def test_to_date_after_1_day
               to = (Date.parse(params["from_date"]) + 1).to_s
-              mock(@client).request(params.merge("to_date" => to)) { jsonl_dummy_responses }
+              mock(@client).request(params.merge("to_date" => to), Client::SMALLSET_BYTE_RANGE) { jsonl_dummy_responses }
 
               @client.export_for_small_dataset(params)
             end
@@ -100,14 +100,14 @@ module Embulk
             def test_to_date_after_1_day_after_10_days_if_empty
               to1 = (Date.parse(params["from_date"]) + 1).to_s
               to2 = (Date.parse(params["from_date"]) + 10).to_s
-              mock(@client).request(params.merge("to_date" => to1)) { "" }
-              mock(@client).request(params.merge("to_date" => to2)) { jsonl_dummy_responses }
+              mock(@client).request(params.merge("to_date" => to1), Client::SMALLSET_BYTE_RANGE) { "" }
+              mock(@client).request(params.merge("to_date" => to2), Client::SMALLSET_BYTE_RANGE) { jsonl_dummy_responses }
 
               @client.export_for_small_dataset(params)
             end
 
             def test_config_error_when_too_long_empty_dates
-              stub(@client).request(anything) { "" }
+              stub(@client).request(anything, anything) { "" }
 
               assert_raise(Embulk::ConfigError) do
                 @client.export_for_small_dataset(params)

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: embulk-input-mixpanel
 version: !ruby/object:Gem::Version
-  version: 0.4.0
+  version: 0.4.1
 platform: ruby
 authors:
 - yoshihara
@@ -9,7 +9,7 @@ authors:
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2016-03-04 00:00:00.000000000 Z
+date: 2016-03-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement