embulk-input-mixpanel 0.5.1 → 0.5.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: a00fda7cfbd7a835af6407887de6be6680178752
-  data.tar.gz: a736976f532f0395b32031b12f86ac13982468a0
+  metadata.gz: 8534316e5eae7127b70afc10d9b247e872643c32
+  data.tar.gz: e629adeb4d42e6386564bd13cfba3bce107ecd9c
 SHA512:
-  metadata.gz: cefb098dc10c716f228385ba644d1743b911b140cc96148188de0c1ab7ac8000ac262ae4a510949eb7de30f37d0219119438c90d19fd701d34c860bd367d507d
-  data.tar.gz: 96e16695df615dacd4d59c563755c73de889e904a10a1dc29248c97a3ed5e40933535f5cf0bebeb7702bd56aefb84d904aed5b151ea4c5aad365c24c8e201438
+  metadata.gz: 70287307438223546321af68362fc63fd56f592eb09d17d543d53986990109bb53b285a8467500617f7a8b99ab4991b6850a7b53c0df241282ee2259f5a8808d
+  data.tar.gz: f8e370285d393fada617ccc2d4dcb9d5cd54396cf47757a9af6f425bdfb83ac77c6354127171a6859939367698cd9b23e0915b8c954e557533140dcc1ea1525c

data/CHANGELOG.md

@@ -1,3 +1,7 @@
+## 0.5.2 - 2017-07-26
+* [enhancement]Enable realtime data export[#47](https://github.com/treasure-data/embulk-input-mixpanel/pull/47)  
+* [maintenance]Fix incorrect error message[#49](https://github.com/treasure-data/embulk-input-mixpanel/pull/49)
+
 ## 0.5.1 - 2016-12-13
 * Enable TCP Keepalive to protect from NAT [#48](https://github.com/treasure-data/embulk-input-mixpanel/pull/48)
 

data/embulk-input-mixpanel.gemspec

@@ -1,7 +1,7 @@
 
 Gem::Specification.new do |spec|
   spec.name          = "embulk-input-mixpanel"
-  spec.version       = "0.5.1"
+  spec.version       = "0.5.2"
   spec.authors       = ["yoshihara", "uu59"]
   spec.summary       = "Mixpanel input plugin for Embulk"
   spec.description   = "Loads records from Mixpanel."

data/lib/embulk/input/mixpanel.rb

@@ -51,6 +51,7 @@ module Embulk
           fetch_custom_properties: config.param(:fetch_custom_properties, :bool, default: true),
           retry_initial_wait_sec: config.param(:retry_initial_wait_sec, :integer, default: 1),
           retry_limit: config.param(:retry_limit, :integer, default: 5),
+          latest_fetched_time: config.param(:latest_fetched_time, :integer, default: 0),
         }
 
         if task[:fetch_unknown_columns] && task[:fetch_custom_properties]
@@ -82,9 +83,12 @@ module Embulk
         # NOTE: If this plugin supports to run by multi threads, this
         # implementation is terrible.
         task_report = task_reports.first
-        next_to_date = Date.parse(task_report[:to_date]).next
+        next_to_date = Date.parse(task_report[:to_date])
 
-        next_config_diff = {from_date: next_to_date.to_s}
+        next_config_diff = {
+          from_date: next_to_date.to_s,
+          latest_fetched_time: task_report[:latest_fetched_time],
+        }
         return next_config_diff
       end
 
@@ -133,13 +137,28 @@ module Embulk
 
       def run
         self.class.giveup_when_mixpanel_is_down
+        prev_latest_fetched_time = task[:latest_fetched_time] || 0
+        prev_latest_fetched_time_format = Time.at(prev_latest_fetched_time).strftime("%F %T %z")
+        current_latest_fetched_time = prev_latest_fetched_time
 
         @dates.each_slice(SLICE_DAYS_COUNT) do |dates|
+          ignored_record_count = 0
           unless preview?
             Embulk.logger.info "Fetching data from #{dates.first} to #{dates.last} ..."
           end
 
           fetch(dates).each do |record|
+            record_time = record["properties"]["time"]
+            if record_time <= prev_latest_fetched_time
+              ignored_record_count += 1
+              next
+            end
+
+            current_latest_fetched_time= [
+              current_latest_fetched_time,
+              record_time,
+            ].max
+
             values = extract_values(record)
             if @fetch_unknown_columns
               unknown_values = extract_unknown_values(record)
@@ -151,12 +170,18 @@ module Embulk
             page_builder.add(values)
           end
 
+          if ignored_record_count > 0
+            Embulk.logger.warn "Skipped already loaded #{ignored_record_count} records. These record times are older or equal than previous fetched record time (#{prev_latest_fetched_time} @ #{prev_latest_fetched_time_format})."
+          end
           break if preview?
         end
 
         page_builder.finish
 
-        task_report = {to_date: @dates.last || (Date.today - 1)}
+        task_report = {
+          latest_fetched_time: current_latest_fetched_time,
+          to_date: @dates.last || Date.today - 1,
+        }
         return task_report
       end
 

data/lib/embulk/input/mixpanel_api/client.rb

@@ -100,18 +100,24 @@ module Embulk
           set_signatures(params)
 
           buf = ""
-          response = httpclient.get(ENDPOINT_EXPORT, params) do |chunk|
-            chunk.each_line do |line|
-              begin
-                record = JSON.parse(buf + line)
-                block.call record
-                buf = ""
-              rescue JSON::ParserError => e
-                buf << line
+          error_response = ''
+          response = httpclient.get(ENDPOINT_EXPORT, params) do |response, chunk|
+            # Only process data if response status is 200..299
+            if response.status/100 == 2
+              chunk.each_line do |line|
+                begin
+                  record = JSON.parse(buf + line)
+                  block.call record
+                  buf = ""
+                rescue JSON::ParserError => e
+                  buf << line
+                end
               end
+            else
+               error_response << chunk
             end
           end
-          handle_error(response)
+          handle_error(response, error_response)
         end
 
         def request_small_dataset(params, range)
@@ -123,21 +129,21 @@ module Embulk
             # cannot satisfied requested Range, get full body
             res = httpclient.get(ENDPOINT_EXPORT, params)
           end
-          handle_error(res)
+          handle_error(res,res.body)
           response_to_enum(res.body)
         end
 
-        def handle_error(response)
+        def handle_error(response, error_response)
           Embulk.logger.debug "response code: #{response.code}"
           case response.code
           when 400..499
             if response.code == 429
               # [429] {"error": "too many export requests in progress for this project"}
-              raise RuntimeError.new("[#{response.code}] #{response.body} (will retry)")
+              raise RuntimeError.new("[#{response.code}] #{error_response} (will retry)")
             end
-            raise ConfigError.new("[#{response.code}] #{response.body}")
+            raise ConfigError.new("[#{response.code}] #{error_response}")
           when 500..599
-            raise RuntimeError.new("[#{response.code}] #{response.body}")
+            raise RuntimeError.new("[#{response.code}] #{error_response}")
           end
         end
 

data/lib/range_generator.rb

@@ -9,7 +9,7 @@ class RangeGenerator
   def generate_range
     validate
     show_warnings
-    range_only_past.map{|date| date.to_s}
+    range_only_present.map{|date| date.to_s}
   end
 
   private
@@ -49,12 +49,12 @@ class RangeGenerator
     if fetch_days
       from_date..(from_date + fetch_days - 1)
     else
-      from_date..yesterday
+      from_date..today
     end
   end
 
-  def range_only_past
-    range.find_all{|date| date < today}
+  def range_only_present
+    range.find_all{|date| date <= today}
   end
 
   def overdays?
@@ -62,15 +62,11 @@ class RangeGenerator
   end
 
   def overdays
-    range.to_a - range_only_past.to_a
+    range.to_a - range_only_present.to_a
   end
 
   def from_date_too_early?
-    from_date > yesterday
-  end
-
-  def yesterday
-    today - 1
+    from_date > today
   end
 
   def today

data/test/embulk/input/test_mixpanel.rb

@@ -72,12 +72,12 @@ module Embulk
           assert_equal(expected, actual)
         end
 
-        def test_from_date_today
+        def test_from_date_future
           config = {
             type: "mixpanel",
             api_key: API_KEY,
             api_secret: API_SECRET,
-            from_date: Date.today.to_s,
+            from_date: (Date.today + 1).to_s,
           }
 
           stub_export_all
@@ -245,7 +245,7 @@ module Embulk
           end
 
           def target_dates
-            dates.find_all{|d| d < Date.today}.map {|date| date.to_s}
+            dates.find_all{|d| d <= Date.today}.map {|date| date.to_s}
           end
 
           def transaction_config
@@ -375,9 +375,9 @@ module Embulk
 
         def test_resume
           today = Date.today
-          control = proc { [{to_date: today.to_s}] }
+          control = proc { [{to_date: today.to_s, latest_fetched_time: 999}] }
           actual = Mixpanel.resume(transaction_task, columns, 1, &control)
-          assert_equal({from_date: today.next.to_s}, actual)
+          assert_equal({from_date: today.to_s, latest_fetched_time: 999}, actual)
         end
 
         def control
@@ -504,6 +504,7 @@ module Embulk
             fetch_custom_properties: false,
             retry_initial_wait_sec: 0,
             retry_limit: 3,
+            latest_fetched_time: 0,
           }
         end
       end
@@ -564,6 +565,7 @@ module Embulk
             added = [
               record["event"],
               record["properties"]["$specified"],
+              record["properties"]["time"] - 32400, # timezone adjust
               custom_property_keys.map{|k| {k => record["properties"][k] }}.inject(&:merge)
             ]
 
@@ -583,6 +585,7 @@ module Embulk
             {
               "event" => "EV",
               "properties" => {
+                "time" => 1000000,
                 "$os" => "Android",
                 "$specified" => "foo",
                 "$foobar" => "foobar",
@@ -594,6 +597,7 @@ module Embulk
             [
               {"name" => "event", "type" => "string"},
               {"name" => "$specified", "type" => "string"},
+              {"name" => "time", "type" => "integer"},
             ]
           end
         end
@@ -674,6 +678,7 @@ module Embulk
           fetch_custom_properties: false,
           retry_initial_wait_sec: 2,
           retry_limit: 3,
+          latest_fetched_time: 0,
         }
       end
 
@@ -709,6 +714,7 @@ module Embulk
           fetch_custom_properties: false,
           retry_initial_wait_sec: 2,
           retry_limit: 3,
+          latest_fetched_time: 0,
         }
       end
 

data/test/test_range_generator.rb

@@ -37,8 +37,8 @@ class RangeGeneratorTest < Test::Unit::TestCase
         @warn_message_regexp = /ignored them/
       end
 
-      def test_range_only_past
-        expected_to = Date.today - 1
+      def test_range_only_present
+        expected_to = Date.today
         expected = (@from..expected_to).to_a.map{|date| date.to_s}
 
         stub(Embulk.logger).warn(@warn_message_regexp)

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: embulk-input-mixpanel
 version: !ruby/object:Gem::Version
-  version: 0.5.1
+  version: 0.5.2
 platform: ruby
 authors:
 - yoshihara
@@ -9,7 +9,7 @@ authors:
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2016-12-13 00:00:00.000000000 Z
+date: 2017-07-26 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement