embulk-input-mixpanel 0.4.2 → 0.4.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 530a912601d813797ba6239f907ddd05d5a877f0
-  data.tar.gz: 4b1c952056474b426fc3e4fedff9a200c9362590
+  metadata.gz: 82bac0d9e5e93a4fc606b3d43f1e9cd35debf4b3
+  data.tar.gz: 82e5c6323b3d9db15ff54589b572b642cb6d93e1
 SHA512:
-  metadata.gz: 581d5574847051fff79c93a2d14df66212ea31a7a8a9f5e3a275250a31c5a902383baf164e5f3c7471d34d34eeb9c1cabd5168d5ed920be4d3e16e0a736f702f
-  data.tar.gz: b679af78c706c0723477136cfe94e51445a5bf3a5b9ec721649f6ed6a0cf74871d6984f490c9a24c47d7d84de9e24623302e8e8611a6a3ef96e0e04181fd7d4f
+  metadata.gz: e606300ab082a98aa443432a0b4f4246c40a942ca7724bb68679d66a889fce60cb36eb95b7dcf59e7a35181bb3175efc785cb3710d7764b58d800209589a140d
+  data.tar.gz: abf1f2528faf56e7c6e65d72d4f8e34a6beebda7f93de1eb16bd93130de2c4b647cf716053ec039a7ddd8b1fdbfa9c62864bb280334630b53af1f15db6137cd3

data/CHANGELOG.md

@@ -1,3 +1,6 @@
+## 0.4.3 - 2016-03-16
+* [enhancement] Custom properties json [#40](https://github.com/treasure-data/embulk-input-mixpanel/pull/40)
+
 ## 0.4.2 - 2016-03-08
 * [fixed] Fix Range request was not satisfied [#39](https://github.com/treasure-data/embulk-input-mixpanel/pull/39)
 

data/README.md

@@ -38,14 +38,57 @@ To get it, you should log in mixpanel website, and click gear icon at the lower
   - NOTE: Mixpanel API supports to export data from at least 2 days before to at most the previous day.
 - **fetch_days**: Count of days range for exporting (integer, optional, default: from_date - (today - 1))
   - NOTE: Mixpanel doesn't support to from_date > today - 2
-- **fetch_unknown_columns**: If you want this plugin fetches unknown (unconfigured in config) columns (boolean, optional, default: true)
+- **fetch_unknown_columns**(deprecated): If you want this plugin fetches unknown (unconfigured in config) columns (boolean, optional, default: true)
   - NOTE: If true, `unknown_columns` column is created and added unknown columns' data.
+- **fetch_custom_properties**: All custom properties into `custom_properties` key. "custom properties" are not desribed Mixpanel document [1](https://mixpanel.com/help/questions/articles/special-or-reserved-properties), [2](https://mixpanel.com/help/questions/articles/what-properties-do-mixpanels-libraries-store-by-default).  (boolean, optional, default: false)
+  - NOTE: Cannot set both `fetch_unknown_columns` and `fetch_custom_properties` to `true`.
 - **event**: The event or events to filter data (array, optional, default: nil)
 - **where**: Expression to filter data (c.f. https://mixpanel.com/docs/api-documentation/data-export-api#segmentation-expressions) (string, optional, default: nil)
 - **bucket**:The data backet to filter data (string, optional, default: nil)
 - **retry_initial_wait_sec** Wait seconds for exponential backoff initial value (integer, default: 1)
 - **retry_limit**: Try to retry this times (integer, default: 5)
 
+### `fetch_unknown_columns` and `fetch_custom_properties`
+
+If you have such data and set config.yml as below.
+
+| event | $city   | $custom | $foobar |
+| ----- | ------- | ------- | ------- |
+| ev    | Tokyo   | custom  | foobar  |
+
+(NOTE: `$city` is a [reserved key](https://mixpanel.com/help/questions/articles/what-properties-do-mixpanels-libraries-store-by-default), `$custom` and `$foobar` are not)
+
+```yaml
+in:
+  type: mixpanel
+  api_key: "API_KEY"
+  api_secret: "API_SECRET"
+  timezone: "US/Pacific"
+  from_date: "2015-07-19"
+  fetch_days: 5
+  columns:
+    - {name: event, type: string}
+    - {name: $custom, type: string}
+```
+
+
+`fetch_unknown_columns: true` will fetch as:
+
+| event | $custom | unknown_columns (json) |
+| ----- | ------- | ----------------- |
+| ev    | custom  | `{"$city":"Tokyo", "$foobar": "foobar"}` |
+
+`fetch_custom_properties: true` will fetch as:
+
+| event | $custom | custom_properties (json) |
+| ----- | ------- | ----------------- |
+| ev    | custom  | `{"$foobar": "foobar"}` |
+
+
+`fetch_unknown_columns` recognize `$city` and `$foobar` as `unknown_columns` because they are not described in config.yml.
+
+`fetch_custom_properties` recognize `$foobar` as `custom_properties`. `$custom` is also custom property but it was described in config.yml.
+
 ## Example
 
 ```yaml

data/Rakefile

@@ -5,7 +5,7 @@ task default: :test
 
 desc "Run tests"
 task :test do
-  ruby("test/run-test.rb", "--use-color=yes", "--collector=dir")
+  ruby("--debug", "test/run-test.rb", "--use-color=yes", "--collector=dir")
 end
 
 Everyleaf::EmbulkHelper::Tasks.install(

data/embulk-input-mixpanel.gemspec

@@ -1,7 +1,7 @@
 
 Gem::Specification.new do |spec|
   spec.name          = "embulk-input-mixpanel"
-  spec.version       = "0.4.2"
+  spec.version       = "0.4.3"
   spec.authors       = ["yoshihara", "uu59"]
   spec.summary       = "Mixpanel input plugin for Embulk"
   spec.description   = "Loads records from Mixpanel."

data/lib/embulk/input/mixpanel.rb

@@ -12,6 +12,18 @@ module Embulk
       GUESS_RECORDS_COUNT = 10
       NOT_PROPERTY_COLUMN = "event".freeze
 
+      # https://mixpanel.com/help/questions/articles/special-or-reserved-properties
+      # https://mixpanel.com/help/questions/articles/what-properties-do-mixpanels-libraries-store-by-default
+      #
+      # JavaScript to extract key names from HTML: run it on Chrome Devtool when opening their document
+      # > Array.from(document.querySelectorAll("strong")).map(function(s){ return s.textContent.match(/[A-Z]/) ? s.parentNode.textContent.match(/\((.*?)\)/)[1] : s.textContent.split(",").join(" ") }).join(" ")
+      # > Array.from(document.querySelectorAll("li")).map(function(s){ m = s.textContent.match(/\((.*?)\)/); return m && m[1] }).filter(function(k) { return k && !k.match("utm") }).join(" ")
+      KNOWN_KEYS = %W(
+        #{NOT_PROPERTY_COLUMN}
+        distinct_id ip mp_name_tag mp_note token time mp_country_code length campaign_id $email $phone $distinct_id $ios_devices $android_devices $first_name  $last_name  $name $city $region $country_code $timezone $unsubscribed
+        $city $region mp_country_code $browser $browser_version $device $current_url $initial_referrer $initial_referring_domain $os $referrer $referring_domain $screen_height $screen_width $search_engine $city $region $mp_country_code $timezone $browser_version $browser $initial_referrer $initial_referring_domain $os $last_seen $city $region mp_country_code $app_release $app_version $carrier $ios_ifa $os_version $manufacturer $lib_version $model $os $screen_height $screen_width $wifi $city $region $mp_country_code $timezone $ios_app_release $ios_app_version $ios_device_model $ios_lib_version $ios_version $ios_ifa $last_seen $city $region mp_country_code $app_version $bluetooth_enabled $bluetooth_version $brand $carrier $has_nfc $has_telephone $lib_version $manufacturer $model $os $os_version $screen_dpi $screen_height $screen_width $wifi $google_play_services $city $region mp_country_code $timezone $android_app_version $android_app_version_code $android_lib_version $android_os $android_os_version $android_brand $android_model $android_manufacturer $last_seen
+      ).uniq.freeze
+
       # NOTE: It takes long time to fetch data between from_date to
       # to_date by one API request. So this plugin fetches data
       # between each 7 (SLICE_DAYS_COUNT) days.
@@ -36,10 +48,15 @@ module Embulk
           api_secret: config.param(:api_secret, :string),
           schema: config.param(:columns, :array),
           fetch_unknown_columns: fetch_unknown_columns,
+          fetch_custom_properties: config.param(:fetch_custom_properties, :bool, default: false),
           retry_initial_wait_sec: config.param(:retry_initial_wait_sec, :integer, default: 1),
           retry_limit: config.param(:retry_limit, :integer, default: 5),
         }
 
+        if task[:fetch_unknown_columns] && task[:fetch_custom_properties]
+          raise Embulk::ConfigError.new("Don't set true both `fetch_unknown_columns` and `fetch_custom_properties`.")
+        end
+
         columns = task[:schema].map do |column|
           name = column["name"]
           type = column["type"].to_sym
@@ -48,9 +65,14 @@ module Embulk
         end
 
         if fetch_unknown_columns
+          Embulk.logger.warn "Deprecated `unknown_columns`. Use `fetch_custom_properties` instead."
           columns << Column.new(nil, "unknown_columns", :json)
         end
 
+        if task[:fetch_custom_properties]
+          columns << Column.new(nil, "custom_properties", :json)
+        end
+
         resume(task, columns, 1, &control)
       end
 
@@ -115,6 +137,9 @@ module Embulk
               unknown_values = extract_unknown_values(record)
               values << unknown_values.to_json
             end
+            if task[:fetch_custom_properties]
+              values << collect_custom_properties(record)
+            end
             page_builder.add(values)
           end
 
@@ -153,6 +178,16 @@ module Embulk
         end
       end
 
+      def collect_custom_properties(record)
+        specified_columns = @schema.map{|col| col["name"]}
+        custom_keys = record["properties"].keys.find_all{|key| !KNOWN_KEYS.include?(key.to_s) && !specified_columns.include?(key.to_s) }
+        custom_keys.inject({}) do |result, key|
+          result.merge({
+            key => record["properties"][key]
+          })
+        end
+      end
+
       def extract_unknown_values(record)
         record_keys = record["properties"].keys + [NOT_PROPERTY_COLUMN]
         schema_keys = @schema.map {|column| column["name"]}

data/test/embulk/input/test_mixpanel.rb

@@ -335,6 +335,44 @@ module Embulk
           end
         end
 
+        class TestCustomProps < self
+          setup do
+            stub(Mixpanel).resume {}
+          end
+
+          data(
+            "false/false" => [false, false],
+            "false/true" => [false, true],
+            "true/false" => [true, false],
+          )
+          def test_valid_combination(data)
+            fetch_unknown_columns, fetch_custom_properties = data
+            conf = DataSource[*transaction_config.merge(fetch_unknown_columns: fetch_unknown_columns, fetch_custom_properties: fetch_custom_properties).to_a.flatten(1)]
+
+            assert_nothing_raised do
+              Mixpanel.transaction(conf, &control)
+            end
+          end
+
+          def test_both_true_then_raise_config_error
+            conf = DataSource[*transaction_config.merge(fetch_unknown_columns: true, fetch_custom_properties: true).to_a.flatten(1)]
+
+            assert_raise(Embulk::ConfigError) do
+              Mixpanel.transaction(conf, &control)
+            end
+          end
+
+          private
+
+          def transaction_config
+            config.merge(
+              columns: schema,
+              fetch_days: 2,
+              timezone: "UTC",
+            )
+          end
+        end
+
         def test_resume
           today = Date.today
           control = proc { [{to_date: today.to_s}] }
@@ -463,6 +501,7 @@ module Embulk
             dates: DATES.to_a.map(&:to_s),
             params: Mixpanel.export_params(embulk_config),
             fetch_unknown_columns: false,
+            fetch_custom_properties: false,
             retry_initial_wait_sec: 0,
             retry_limit: 3,
           }
@@ -509,6 +548,56 @@ module Embulk
           @plugin.run
         end
 
+        class CustomPropertiesTest < self
+          def setup
+            super
+            @page_builder = Object.new
+            @plugin = Mixpanel.new(task, nil, nil, @page_builder)
+            stub(@plugin).fetch { [record] }
+          end
+
+          def test_run
+            stub(@plugin).preview? { false }
+
+            custom_property_keys = %w($foobar)
+
+            added = [
+              record["event"],
+              record["properties"]["$specified"],
+              custom_property_keys.map{|k| {k => record["properties"][k] }}.inject(&:merge)
+            ]
+
+            mock(@page_builder).add(added).at_least(1)
+            mock(@page_builder).finish
+
+            @plugin.run
+          end
+
+          private
+
+          def task
+            super.merge(schema: schema, fetch_unknown_columns: false, fetch_custom_properties: true)
+          end
+
+          def record
+            {
+              "event" => "EV",
+              "properties" => {
+                "$os" => "Android",
+                "$specified" => "foo",
+                "$foobar" => "foobar",
+              }
+            }
+          end
+
+          def schema
+            [
+              {"name" => "event", "type" => "string"},
+              {"name" => "$specified", "type" => "string"},
+            ]
+          end
+        end
+
         class UnknownColumnsTest < self
           def setup
             super
@@ -581,6 +670,7 @@ module Embulk
           dates: DATES.to_a.map(&:to_s),
           params: Mixpanel.export_params(embulk_config),
           fetch_unknown_columns: false,
+          fetch_custom_properties: false,
           retry_initial_wait_sec: 2,
           retry_limit: 3,
         }
@@ -615,6 +705,7 @@ module Embulk
           from_date: FROM_DATE,
           fetch_days: DAYS,
           fetch_unknown_columns: false,
+          fetch_custom_properties: false,
           retry_initial_wait_sec: 2,
           retry_limit: 3,
         }

data/test/run-test.rb

@@ -13,6 +13,12 @@ $LOAD_PATH.unshift(test_dir)
 
 ENV["TEST_UNIT_MAX_DIFF_TARGET_STRING_SIZE"] ||= "5000"
 
-CodeClimate::TestReporter.start
+if ENV["CI"]
+  require "codeclimate-test-reporter"
+  CodeClimate::TestReporter.start
+elsif ENV["COVERAGE"]
+  require 'simplecov'
+  SimpleCov.start 'test_frameworks'
+end
 
 exit Test::Unit::AutoRunner.run(true, test_dir)

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: embulk-input-mixpanel
 version: !ruby/object:Gem::Version
-  version: 0.4.2
+  version: 0.4.3
 platform: ruby
 authors:
 - yoshihara
@@ -9,7 +9,7 @@ authors:
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2016-03-08 00:00:00.000000000 Z
+date: 2016-03-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement