embulk-input-mixpanel 0.5.3.alpha.1 → 0.5.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 28a4798aa8352ee4fbf06a23a57ff44aeb4cb902
-  data.tar.gz: 433333137ed26c3a2d62b822b767621885095cf4
+  metadata.gz: e5d08021cb9b3e4518546d44eeff78eab3a6e344
+  data.tar.gz: 55b1de01d58db4f50755996917f7f40fdc0fdd33
 SHA512:
-  metadata.gz: 2d5fbba923cf5e52c169ed94001633001a2fcd8bb83cf0138541581d5a7768e4fd51bf7d690f7105c3fb31cf480a27a1af4fa9633636c8e4a237ba5786c99722
-  data.tar.gz: a147657608530a5f0ef513fdd150e93cf62bee74abeeadc3046ad30df42d19c1b94c4f42606260c827f1785dfc3c5c9fef6cf231efbc55ef4351d7be38cd4002
+  metadata.gz: 10814321b400f21bfc54acb2005c671265e0fb5a803ab221c7fa5a9dbb02a513a18ea144a1e4dbf9fa92c964b71ca7655f86a2ff866c106df6a5079628b5d3d3
+  data.tar.gz: 85adb7eeac61dec517c2010300e8fa1a58eb0159c86fbe17d0b34ab12209e774fa15ce4bd425f79d495a087519554b46d0eb07627d03d8e601697b2fa212e275

data/CHANGELOG.md

@@ -1,3 +1,9 @@
+## 0.5.3 - 2017-08-07
+* [enhancement] Allow user to choose to run incremental or not `incremental` option default to true [#50](https://github.com/treasure-data/embulk-input-mixpanel/pull/50)
+* [enhancement] Allow user to specify an `incremental_column`, which will be add to the where praramter is API requests[#50](https://github.com/treasure-data/embulk-input-mixpanel/pull/50)
+* [enhancement] Allow user to specifiy backfill days, this `back_fill_time` option will tell plugin how many days we look back for data [#50](https://github.com/treasure-data/embulk-input-mixpanel/pull/50)
+
+
 ## 0.5.2 - 2017-07-26
 * [enhancement]Enable realtime data export[#47](https://github.com/treasure-data/embulk-input-mixpanel/pull/47)  
 * [maintenance]Fix incorrect error message[#49](https://github.com/treasure-data/embulk-input-mixpanel/pull/49)

data/README.md

@@ -38,6 +38,10 @@ To get it, you should log in mixpanel website, and click gear icon at the lower
   - NOTE: Mixpanel API supports to export data from at least 2 days before to at most the previous day.
 - **fetch_days**: Count of days range for exporting (integer, optional, default: from_date - (today - 1))
   - NOTE: Mixpanel doesn't support to from_date > today - 2
+- **incremental**: Run incremental mode nor not (boolean, optional, default: true)
+- **incremental_column**: Column to be add to where query as a constraint for incremental time. Only data that have incremental_column timestamp > than previous latest_fetched_time will be return (string, optional, default: nil)
+- **back_fill_time**: Amount of time that will be subtracted from `from_date` to calculate the final `from_date` that will be use for API Request. This is due to Mixpanel caching data on user devices before sending it to Mixpanel server (integer, optional, default: 5)
+  - NOTE: Only have effect when incremental is true and incremental_column is specified
 - **fetch_unknown_columns**(deprecated): If you want this plugin fetches unknown (unconfigured in config) columns (boolean, optional, default: false)
   - NOTE: If true, `unknown_columns` column is created and added unknown columns' data.
 - **fetch_custom_properties**: All custom properties into `custom_properties` key. "custom properties" are not desribed Mixpanel document [1](https://mixpanel.com/help/questions/articles/special-or-reserved-properties), [2](https://mixpanel.com/help/questions/articles/what-properties-do-mixpanels-libraries-store-by-default).  (boolean, optional, default: true)

data/embulk-input-mixpanel.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |spec|
   spec.name          = "embulk-input-mixpanel"
-  spec.version       = "0.5.3.alpha.1"
+  spec.version       = "0.5.3"
   spec.authors       = ["yoshihara", "uu59"]
   spec.summary       = "Mixpanel input plugin for Embulk"
   spec.description   = "Loads records from Mixpanel."

data/lib/embulk/input/mixpanel.rb

@@ -46,9 +46,9 @@ module Embulk
         latest_fetched_time = config.param(:latest_fetched_time, :integer, default: 0)
 
         # Backfill from date if incremental and an incremental field is set and we are in incremental run
-        if incremental && !incremental_column.nil? && latest_fetched_time !=0
+        if incremental && incremental_column && latest_fetched_time !=0
           back_fill_days = config.param(:back_fill_days, :integer, default: 5)
-          puts "Backfill days #{back_fill_days}"
+          Embulk.logger.info "Backfill days #{back_fill_days}"
           from_date = (Date.parse(from_date) - back_fill_days).to_s
           fetch_days = fetch_days.nil? ? nil : fetch_days + back_fill_days
         end
@@ -167,24 +167,24 @@ module Embulk
             Embulk.logger.info "Fetching data from #{dates.first} to #{dates.last} ..."
           end
           record_time_column=@incremental_column || DEFAULT_TIME_COLUMN
-          fetch(dates,prev_latest_fetched_time).each do |record|
+          fetch(dates, prev_latest_fetched_time).each do |record|
             if @incremental
               if !record["properties"].include?(record_time_column)
                 raise Embulk::ConfigError.new("Incremental column not exists in fetched data #{record_time_column}")
               end
               record_time = record["properties"][record_time_column]
-             if @incremental_column.nil?
-               if record_time <= prev_latest_fetched_time
-                 ignored_record_count += 1
-                 next
-               end
-             end
-
-             current_latest_fetched_time= [
-               current_latest_fetched_time,
-               record_time,
-             ].max
-           end
+              if @incremental_column.nil?
+                if record_time <= prev_latest_fetched_time
+                  ignored_record_count += 1
+                  next
+                end
+              end
+
+              current_latest_fetched_time= [
+                current_latest_fetched_time,
+                record_time,
+              ].max
+            end
             values = extract_values(record)
             if @fetch_unknown_columns
               unknown_values = extract_unknown_values(record)
@@ -272,7 +272,7 @@ module Embulk
             "where" => "#{params['where'].nil? ? '' : "(#{params['where']}) and " }properties[\"#{@incremental_column}\"] > #{last_fetch_time}"
           )
         end
-        puts "Where params is #{params["where"]}"
+        Embulk.logger.info "Where params is #{params["where"]}"
         client = MixpanelApi::Client.new(@api_key, @api_secret, self.class.perfect_retry(task))
 
         if preview?

data/test/embulk/input/test_mixpanel.rb

@@ -616,14 +616,14 @@ module Embulk
 
           def test_incremental_column_with_where
             page_builder = Object.new
-            plugin = Mixpanel.new(task.merge(params: task[:params].merge("where" => "abc==def")), nil, nil, page_builder)
+            plugin = Mixpanel.new(task.merge(params: task[:params].merge("where" => "abc==def"),latest_fetched_time: 1), nil, nil, page_builder)
             stub(plugin).preview? {false}
             adjusted = record_epoch - timezone_offset_seconds
             mock(page_builder).add(["FOO", adjusted, "event"]).times(records.length * 2)
             mock(page_builder).finish
             any_instance_of(MixpanelApi::Client) do |klass|
               stub(klass).export() do |params, block|
-                assert_equal('(abc==def) and properties["mp_processing_time_ms"] > 0',params["where"])
+                assert_equal('(abc==def) and properties["mp_processing_time_ms"] > 1',params["where"])
                 records.each{|record| block.call(record) }
               end
             end
@@ -633,14 +633,14 @@ module Embulk
 
           def test_incremental_column
             page_builder = Object.new
-            plugin = Mixpanel.new(task, nil, nil, page_builder)
+            plugin = Mixpanel.new(task.merge(latest_fetched_time: 1), nil, nil, page_builder)
             stub(plugin).preview? {false}
             adjusted = record_epoch - timezone_offset_seconds
             mock(page_builder).add(["FOO", adjusted, "event"]).times(records.length * 2)
             mock(page_builder).finish
             any_instance_of(MixpanelApi::Client) do |klass|
               stub(klass).export() do |params, block|
-                assert_equal('properties["mp_processing_time_ms"] > 0',params["where"])
+                assert_equal('properties["mp_processing_time_ms"] > 1',params["where"])
                 records.each{|record| block.call(record) }
               end
             end

data/test/prepare_embulk.rb

@@ -1,6 +1,4 @@
 module PrepareEmbulk
-  require "embulk/command/embulk_run"
-
   if Embulk.respond_to?(:home)
     # keep compatibility for Embulk 0.6.x
     classpath_dir = Embulk.home("classpath")

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: embulk-input-mixpanel
 version: !ruby/object:Gem::Version
-  version: 0.5.3.alpha.1
+  version: 0.5.3
 platform: ruby
 authors:
 - yoshihara
@@ -9,7 +9,7 @@ authors:
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2017-08-02 00:00:00.000000000 Z
+date: 2017-08-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -219,12 +219,12 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">"
+  - - ">="
     - !ruby/object:Gem::Version
-      version: 1.3.1
+      version: '0'
 requirements: []
 rubyforge_project:
-rubygems_version: 2.6.12
+rubygems_version: 2.4.8
 signing_key:
 specification_version: 4
 summary: Mixpanel input plugin for Embulk