embulk-input-mixpanel 0.5.12 → 0.5.13

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 4c4d29865b7b17b4df43e0b6810a73392ab57627
-  data.tar.gz: 2884b91def9e5e6dbf7ed0eb331a5c9f0355521c
+  metadata.gz: 66b8edacf57562c25bb7663220b5c2f85e585da4
+  data.tar.gz: d631d41e6761ba3dc840085d24d9666a0f0b1221
 SHA512:
-  metadata.gz: cb95e825d05e33d626edf85d1b1760481c6350deb0741b801d5e07afe17c42fd20f0eca7155093b47e6e4e42ba67c1cf7771628210c28ca4ba33893301029a56
-  data.tar.gz: 91792f7c85c26bf04fd88e8874e5ce6472464beb6cfa65e837178eaee2ba478dbf5363c5bb60042081b2a1a83b267934c8487ba91774c27f20f385278e2d73c2
+  metadata.gz: 38cdaa6706bf5ac3b76eb7be40a52ff2473b6e4fe3f7037f9d00ee61cfe35c94d0bd85a3a00d15a1d6486100c5ab1851aee1bce4c54cbeaadc93f90b2396523e
+  data.tar.gz: 0c778717481718ecf4b97c636ec0f5ed6966be18b8687ba0c95941e66726fd4fe543674490a266d5d9948ac31b775574e229030f4efd26cc1159e8e14d9b8acc

data/CHANGELOG.md

@@ -1,3 +1,7 @@
+## 0.5.13 - 2018-10-04
+
+* [enhancement] Limit number of returned records in guess and preview [#60](https://github.com/treasure-data/embulk-input-mixpanel/pull/60)
+
 ## 0.5.12 - 2017-02-09
 
 * [bug] Fix incorrect from_date [#59](https://github.com/treasure-data/embulk-input-mixpanel/pull/59)

data/embulk-input-mixpanel.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |spec|
   spec.name          = "embulk-input-mixpanel"
-  spec.version       = "0.5.12"
+  spec.version       = "0.5.13"
   spec.authors       = ["yoshihara", "uu59"]
   spec.summary       = "Mixpanel input plugin for Embulk"
   spec.description   = "Loads records from Mixpanel."

data/lib/embulk/input/mixpanel.rb

@@ -11,7 +11,6 @@ module Embulk
     class Mixpanel < InputPlugin
       Plugin.register_input("mixpanel", self)
 
-      GUESS_RECORDS_COUNT = 10
       NOT_PROPERTY_COLUMN = "event".freeze
 
       # https://mixpanel.com/help/questions/articles/special-or-reserved-properties
@@ -355,7 +354,7 @@ module Embulk
       end
 
       def self.guess_from_records(records)
-        sample_props = records.first(GUESS_RECORDS_COUNT).map {|r| r["properties"]}
+        sample_props = records.map {|r| r["properties"]}
         schema = Guess::SchemaGuess.from_hash_records(sample_props)
         columns = schema.map do |col|
           next if col.name == "time"

data/lib/embulk/input/mixpanel_api/client.rb

@@ -12,7 +12,7 @@ module Embulk
         PING_TIMEOUT_SECONDS = 3
         PING_RETRY_LIMIT = 3
         PING_RETRY_WAIT = 2
-        SMALLSET_BYTE_RANGE = "0-#{5 * 1024 * 1024}"
+        SMALL_NUM_OF_RECORDS = 10
         DEFAULT_EXPORT_ENDPOINT = "https://data.mixpanel.com/api/2.0/export/".freeze
 
         attr_reader :retryer
@@ -66,7 +66,7 @@ module Embulk
             latest_tried_to_date = to_date
             params["to_date"] = to_date.strftime("%Y-%m-%d")
             records = retryer.with_retry do
-              request_small_dataset(params, SMALLSET_BYTE_RANGE)
+              request_small_dataset(params, SMALL_NUM_OF_RECORDS)
             end
             next if records.first.nil?
             return records
@@ -129,16 +129,13 @@ module Embulk
           end
         end
 
-        def request_small_dataset(params, range)
+        def request_small_dataset(params, num_of_records)
           # guess/preview
-          # Try to fetch first `range` bytes
+          # Try to fetch first number of records
+          params["limit"] = num_of_records
           set_signatures(params)
           Embulk.logger.info "Sending request to #{@endpoint}"
-          res = httpclient.get(@endpoint, params, {"Range" => "bytes=#{range}"})
-          if res.code == 416
-            # cannot satisfied requested Range, get full body
-            res = httpclient.get(@endpoint, params)
-          end
+          res = httpclient.get(@endpoint, params)
           handle_error(res,res.body)
           response_to_enum(res.body)
         end

data/test/embulk/input/mixpanel_api/test_client.rb

@@ -145,7 +145,7 @@ module Embulk
           class ExportSmallDataset < self
             def test_to_date_after_1_day
               to = (Date.parse(params["from_date"]) + 1).to_s
-              mock(@client).request_small_dataset(params.merge("to_date" => to), Client::SMALLSET_BYTE_RANGE) { [:foo] }
+              mock(@client).request_small_dataset(params.merge("to_date" => to), Client::SMALL_NUM_OF_RECORDS) { [:foo] }
 
               @client.export_for_small_dataset(params)
             end
@@ -163,8 +163,8 @@ module Embulk
               stub_client
               to1 = (Date.parse(params["from_date"]) + 1).to_s
               to2 = (Date.parse(params["from_date"]) + 10).to_s
-              mock(@client).request_small_dataset(params.merge("to_date" => to1), Client::SMALLSET_BYTE_RANGE) { [] }
-              mock(@client).request_small_dataset(params.merge("to_date" => to2), Client::SMALLSET_BYTE_RANGE) { [:foo] }
+              mock(@client).request_small_dataset(params.merge("to_date" => to1), Client::SMALL_NUM_OF_RECORDS) { [] }
+              mock(@client).request_small_dataset(params.merge("to_date" => to2), Client::SMALL_NUM_OF_RECORDS) { [:foo] }
 
               @client.export_for_small_dataset(params)
             end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: embulk-input-mixpanel
 version: !ruby/object:Gem::Version
-  version: 0.5.12
+  version: 0.5.13
 platform: ruby
 authors:
 - yoshihara
@@ -9,7 +9,7 @@ authors:
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2018-02-09 00:00:00.000000000 Z
+date: 2018-10-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -239,7 +239,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project:
-rubygems_version: 2.6.6
+rubygems_version: 2.4.8
 signing_key:
 specification_version: 4
 summary: Mixpanel input plugin for Embulk