embulk-input-mixpanel 0.4.3 → 0.4.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 82bac0d9e5e93a4fc606b3d43f1e9cd35debf4b3
-  data.tar.gz: 82e5c6323b3d9db15ff54589b572b642cb6d93e1
+  metadata.gz: f5cb5156064ae0192f6d1d2321a173cd12ef5fd3
+  data.tar.gz: 2a2f3a63e55035bb8dd96a8edf27eb45e546866d
 SHA512:
-  metadata.gz: e606300ab082a98aa443432a0b4f4246c40a942ca7724bb68679d66a889fce60cb36eb95b7dcf59e7a35181bb3175efc785cb3710d7764b58d800209589a140d
-  data.tar.gz: abf1f2528faf56e7c6e65d72d4f8e34a6beebda7f93de1eb16bd93130de2c4b647cf716053ec039a7ddd8b1fdbfa9c62864bb280334630b53af1f15db6137cd3
+  metadata.gz: 48d7e7af0b5ad28fc030e7c1baaa4c34fd236b60fb34df7e8cc1b764825c143aadcab7725d4d78afe3b1ba0ffb05067418217872d79cc0a1d00af7440e7abd90
+  data.tar.gz: b001c794683c39d69ae6b29782988ee3c64ea17a9449d29fb7d86342b144854116116b1928946f33691eaaf0f3ed6d711f985fbe7e4a414c7579a3fd6baf38ec

data/CHANGELOG.md

@@ -1,3 +1,6 @@
+## 0.4.4 - 2016-09-02
+* [enhancement] Reduce memory usage by streaming processing [#42](https://github.com/treasure-data/embulk-input-mixpanel/pull/42)
+
 ## 0.4.3 - 2016-03-16
 * [enhancement] Custom properties json [#40](https://github.com/treasure-data/embulk-input-mixpanel/pull/40)
 

data/embulk-input-mixpanel.gemspec

@@ -1,7 +1,7 @@
 
 Gem::Specification.new do |spec|
   spec.name          = "embulk-input-mixpanel"
-  spec.version       = "0.4.3"
+  spec.version       = "0.4.4"
   spec.authors       = ["yoshihara", "uu59"]
   spec.summary       = "Mixpanel input plugin for Embulk"
   spec.description   = "Loads records from Mixpanel."
@@ -15,7 +15,7 @@ Gem::Specification.new do |spec|
 
   spec.add_dependency 'httpclient'
   spec.add_dependency 'tzinfo'
-  spec.add_dependency 'perfect_retry', ["~> 0.3"]
+  spec.add_dependency 'perfect_retry', ["~> 0.5"]
   spec.add_development_dependency 'bundler', ['~> 1.0']
   spec.add_development_dependency 'rake', ['>= 10.0']
   spec.add_development_dependency 'embulk', ['>= 0.8.6', '< 1.0']

data/lib/embulk/input/mixpanel.rb

@@ -203,20 +203,22 @@ module Embulk
         end
       end
 
-      def fetch(dates)
+      def fetch(dates, &block)
         from_date = dates.first
         to_date = dates.last
         params = @params.merge(
           "from_date" => from_date,
           "to_date" => to_date,
         )
-        client = MixpanelApi::Client.new(@api_key, @api_secret)
+        client = MixpanelApi::Client.new(@api_key, @api_secret, @retryer)
 
-        @retryer.with_retry do
-          if preview?
-            client.export_for_small_dataset(params)
-          else
-            client.export(params)
+        if preview?
+          client.export_for_small_dataset(params)
+        else
+          Enumerator.new do |y|
+            client.export(params) do |record|
+              y << record
+            end
           end
         end
       end

data/lib/embulk/input/mixpanel_api/client.rb

@@ -14,6 +14,8 @@ module Embulk
         PING_RETRY_WAIT = 2
         SMALLSET_BYTE_RANGE = "0-#{5 * 1024 * 1024}"
 
+        attr_reader :retryer
+
         def self.mixpanel_available?
           retryer = PerfectRetry.new do |config|
             config.limit = PING_RETRY_LIMIT
@@ -34,31 +36,42 @@ module Embulk
           end
         end
 
-        def initialize(api_key, api_secret)
+        def initialize(api_key, api_secret, retryer = nil)
           @api_key = api_key
           @api_secret = api_secret
+          @retryer = retryer || PerfectRetry.new do |config|
+            # for test
+            config.limit = 0
+            config.dont_rescues = [RuntimeError]
+            config.log_level = nil
+            config.logger = Embulk.logger
+            config.raise_original_error = true
+          end
         end
 
-        def export(params = {})
-          body = request(params)
-          response_to_enum(body)
+        def export(params = {}, &block)
+          retryer.with_retry do
+            request(params, &block)
+          end
         end
 
-        def export_for_small_dataset(params = {}, times = 0)
-          days = (1 * (10 ** times))
-          to_date = Date.parse(params["from_date"].to_s) + days
-          params["to_date"] = to_date.strftime("%Y-%m-%d")
+        def export_for_small_dataset(params = {})
+          try_to_dates = 5.times.map do |n|
+            # from_date + 1, from_date + 10, from_date + 100, ... so on
+            days = 1 * (10 ** n)
+            Date.parse(params["from_date"].to_s) + days
+          end
 
-          body = request(params, SMALLSET_BYTE_RANGE)
-          result = response_to_enum(body)
-          if result.first.nil?
-            if times >= 5
-              raise ConfigError.new "#{params["from_date"]} + #{days} days has no record. too old date?"
+          try_to_dates.each do |to_date|
+            params["to_date"] = to_date.strftime("%Y-%m-%d")
+            records = retryer.with_retry do
+              request_small_dataset(params, SMALLSET_BYTE_RANGE)
             end
-            export_for_small_dataset(params, times + 1)
-          else
-            result
+            next if records.first.nil?
+            return records
           end
+
+          raise ConfigError.new "#{params["from_date"]}..#{try_to_dates.last} has no record. too old date?"
         end
 
         private
@@ -72,34 +85,53 @@ module Embulk
           end
         end
 
-        def request(params, range = nil)
+        def request(params, &block)
           # https://mixpanel.com/docs/api-documentation/exporting-raw-data-you-inserted-into-mixpanel
-          params[:expire] ||= Time.now.to_i + TIMEOUT_SECONDS
-          params[:sig] = signature(params)
           Embulk.logger.debug "Export param: #{params.to_s}"
+          set_signatures(params)
 
-          headers = {}
-          response =
-            if range
-              # guess/preview
-              res = httpclient.get(ENDPOINT_EXPORT, params, {"Range" => "bytes=#{range}"})
-              if res.code == 416
-                # cannot satisfied requested Range, get full body
-                httpclient.get(ENDPOINT_EXPORT, params)
-              else
-                res
+          buf = ""
+          response = httpclient.get(ENDPOINT_EXPORT, params) do |chunk|
+            chunk.each_line do |line|
+              begin
+                record = JSON.parse(buf + line)
+                block.call record
+                buf = ""
+              rescue JSON::ParserError => e
+                buf << line
               end
-            else
-              httpclient.get(ENDPOINT_EXPORT, params)
             end
+          end
+          handle_error(response)
+        end
+
+        def request_small_dataset(params, range)
+          # guess/preview
+          # Try to fetch first `range` bytes
+          set_signatures(params)
+          res = httpclient.get(ENDPOINT_EXPORT, params, {"Range" => "bytes=#{range}"})
+          if res.code == 416
+            # cannot satisfied requested Range, get full body
+            res = httpclient.get(ENDPOINT_EXPORT, params)
+          end
+          handle_error(res)
+          response_to_enum(res.body)
+        end
+
+        def handle_error(response)
           Embulk.logger.debug "response code: #{response.code}"
           case response.code
           when 400..499
-            raise ConfigError.new response.body
+            raise ConfigError.new("[#{response.code}] #{response.body}")
           when 500..599
-            raise RuntimeError, response.body
+            raise RuntimeError.new("[#{response.code}] #{response.body}")
           end
-          response.body
+        end
+
+        def set_signatures(params)
+          params[:expire] ||= Time.now.to_i + TIMEOUT_SECONDS
+          params[:sig] = signature(params)
+          params
         end
 
         def signature(params)
@@ -121,6 +153,7 @@ module Embulk
               client = HTTPClient.new
               client.receive_timeout = TIMEOUT_SECONDS
               client.default_header = {Accept: "application/json; charset=UTF-8"}
+              # client.debug_dev = STDERR
               client
             end
         end

data/test/embulk/input/mixpanel_api/test_client.rb

@@ -37,38 +37,17 @@ module Embulk
             @httpclient = HTTPClient.new
           end
 
-          def test_httpclient
-            stub_response(success_response)
-            mock(@client).httpclient { @httpclient }
-
-            @client.export(params)
-          end
-
-          def test_response_class
-            stub_client
-            stub_response(success_response)
-
-            actual = @client.export(params)
-
-            assert_equal(Enumerator, actual.class)
-          end
-
-          def test_http_request
-            stub_client
-            mock(@httpclient).get(Client::ENDPOINT_EXPORT, params) do
-              success_response
-            end
-
-            @client.export(params)
-          end
-
           def test_success
             stub_client
+            stub(@client).set_signatures(anything) {}
             stub_response(success_response)
 
-            actual = @client.export(params)
+            records = []
+            @client.export(params) do |record|
+              records << record
+            end
 
-            assert_equal(dummy_responses, actual.to_a)
+            assert_equal(dummy_responses, records)
           end
 
           def test_failure_with_400
@@ -92,16 +71,17 @@ module Embulk
           class ExportSmallDataset < self
             def test_to_date_after_1_day
               to = (Date.parse(params["from_date"]) + 1).to_s
-              mock(@client).request(params.merge("to_date" => to), Client::SMALLSET_BYTE_RANGE) { jsonl_dummy_responses }
+              mock(@client).request_small_dataset(params.merge("to_date" => to), Client::SMALLSET_BYTE_RANGE) { [:foo] }
 
               @client.export_for_small_dataset(params)
             end
 
             def test_to_date_after_1_day_after_10_days_if_empty
+              stub_client
               to1 = (Date.parse(params["from_date"]) + 1).to_s
               to2 = (Date.parse(params["from_date"]) + 10).to_s
-              mock(@client).request(params.merge("to_date" => to1), Client::SMALLSET_BYTE_RANGE) { "" }
-              mock(@client).request(params.merge("to_date" => to2), Client::SMALLSET_BYTE_RANGE) { jsonl_dummy_responses }
+              mock(@client).request_small_dataset(params.merge("to_date" => to1), Client::SMALLSET_BYTE_RANGE) { [] }
+              mock(@client).request_small_dataset(params.merge("to_date" => to2), Client::SMALLSET_BYTE_RANGE) { [:foo] }
 
               @client.export_for_small_dataset(params)
             end
@@ -122,9 +102,12 @@ module Embulk
           end
 
           def stub_response(response)
-            stub(@httpclient).get(Client::ENDPOINT_EXPORT, params) do
-              response
-            end
+            @httpclient.test_loopback_http_response << [
+              "HTTP/1.1 #{response.code}",
+              "Content-Type: application/json",
+              "",
+              response.body
+            ].join("\r\n")
           end
 
           def success_response

data/test/embulk/input/test_mixpanel.rb

@@ -510,17 +510,17 @@ module Embulk
 
       class RunTest < self
         def setup_client
-
           any_instance_of(MixpanelApi::Client) do |klass|
-            stub(klass).request { records_raw_response }
+            stub(klass).request_small_dataset { records_raw_response }
+            stub(klass).request { records }
           end
         end
 
         def setup
           super
-
           @page_builder = Object.new
           @plugin = Mixpanel.new(task, nil, nil, @page_builder)
+          stub(@plugin).fetch { records }
         end
 
         def test_preview
@@ -542,7 +542,7 @@ module Embulk
         def test_timezone
           stub(@plugin).preview? { false }
           adjusted = record_epoch - timezone_offset_seconds
-          mock(@page_builder).add(["FOO", adjusted]).times(records.length * 2)
+          mock(@page_builder).add(["FOO", adjusted, "event"]).times(records.length * 2)
           mock(@page_builder).finish
 
           @plugin.run
@@ -600,13 +600,14 @@ module Embulk
 
         class UnknownColumnsTest < self
           def setup
-            super
             @page_builder = Object.new
             @plugin = Mixpanel.new(task, nil, nil, @page_builder)
+            stub(@plugin).fetch { records }
           end
 
           def test_run
-            Embulk.logger.warn(anything)
+            stub(Embulk.logger).warn
+            stub(Embulk.logger).info
             stub(@plugin).preview? { false }
 
             # NOTE: Expect records are contained same record

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: embulk-input-mixpanel
 version: !ruby/object:Gem::Version
-  version: 0.4.3
+  version: 0.4.4
 platform: ruby
 authors:
 - yoshihara
@@ -9,7 +9,7 @@ authors:
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2016-03-16 00:00:00.000000000 Z
+date: 2016-09-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -44,7 +44,7 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.3'
+        version: '0.5'
   name: perfect_retry
   prerelease: false
   type: :runtime
@@ -52,7 +52,7 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.3'
+        version: '0.5'
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements: