embulk-input-mixpanel 0.5.15 → 0.6.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 6c2a00332d19da50d63a726cac679e745206ccab
-  data.tar.gz: f2810c25272ee6b001f206dd0ddd93895c247ab6
+  metadata.gz: 8fc75a11eeef6fc6e9831e11a5b3eeae414d7799
+  data.tar.gz: 8176825bc52368dd5abd8eea00f394dec3061d37
 SHA512:
-  metadata.gz: 062a5d9d8948ba9efe802b477756484f03add6ce542d9bdbf40ee3988c2c943f5f6445fc32b15d05dfd0b909761fbf4f47f530e0e2d1357b1a57ca0c20074c55
-  data.tar.gz: df58ccf3ab6acad568681d57c39e97258849dc6e3946f5367ccdf31d64265d8c6d6afbb724259b66d21003b670b1763b6e097a0e871ff9bdaeb9e396f5e1b0cc
+  metadata.gz: 2d13fa15449b9900e4179260b6609a536e058de90e96ae691a88ebd757a13b6ddd8972a2715802a8d18557e30157220d1703b933e97a32e2d7766c347fc09c05
+  data.tar.gz: 42c5a4bada8075467e6b1195c7a232f78dc77e71f65a9b573b5e17152af53405a11595f43b4f6456db6f45b54267668a8ed6a6dc6c6db5e59bceb8fb78fec6fb

data/CHANGELOG.md

@@ -1,3 +1,11 @@
+## 0.6.1 - 2020-04-06
+
+* [enhancement] Support JQL script for Profile [#66](https://github.com/treasure-data/embulk-input-mixpanel/pull/66)
+
+## 0.6.0 - 2020-03-30
+
+* [enhancement] Support JQL script [#65](https://github.com/treasure-data/embulk-input-mixpanel/pull/65)
+
 ## 0.5.15 - 2020-01-22
 
 * [enhancement] Update the authentication method to latest [#63](https://github.com/treasure-data/embulk-input-mixpanel/pull/63)

data/README.md

@@ -33,13 +33,16 @@ To get it, you should log in mixpanel website, and click gear icon at the lower
 
 - **api_secret**: project API Secret (string, required)
 - **export_endpoint**: the Data Export API's endpoint (string, default to "http://data.mixpanel.com/api/2.0/export")
+- **jql_endpoint**: the JQL API's endpoint (string, default to "https://mixpanel.com/api/2.0/jql/")
+- **jql_mode**: using JQL or export endpoint (boolean, default to false)
+- **jql_script**: JQL script sent the JQL endpoint(string)
 - **timezone**: project timezone(string, required)
 - **from_date**: From date to export (string, optional, default: today - 2)
   - NOTE: Mixpanel API supports to export data from at least 2 days before to at most the previous day.
 - **fetch_days**: Count of days range for exporting (integer, optional, default: from_date - (today - 1))
   - NOTE: Mixpanel doesn't support to from_date > today - 2
 - **incremental**: Run incremental mode nor not (boolean, optional, default: true)
-- **incremental_column**: Column to be add to where query as a constraint for incremental time. Only data that have incremental_column timestamp > than previous latest_fetched_time will be return (string, optional, default: nil)
+- **incremental_column**: Column to be add to where query as a constraint for incremental time. Only data that have incremental_column timestamp > than previous latest_fetched_time will be return (string, optional, default: time)
 - **back_fill_time**: Amount of time that will be subtracted from `from_date` to calculate the final `from_date` that will be use for API Request. This is due to Mixpanel caching data on user devices before sending it to Mixpanel server (integer, optional, default: 5)
   - NOTE: Only have effect when incremental is true and incremental_column is specified
 - **incremental_column_upper_limit_delay_in_seconds**: When query with incremental column, plugin will lock the upper limit of incremental column query with the job start time, in order to avoid issue with data that commit when the job is running

data/embulk-input-mixpanel.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |spec|
   spec.name          = "embulk-input-mixpanel"
-  spec.version       = "0.5.15"
+  spec.version       = "0.6.1"
   spec.authors       = ["yoshihara", "uu59"]
   spec.summary       = "Mixpanel input plugin for Embulk"
   spec.description   = "Loads records from Mixpanel."

data/lib/embulk/input/mixpanel.rb

@@ -1,88 +1,16 @@
-require "tzinfo"
-require "perfect_retry"
-require "embulk/input/mixpanel_api/client"
-require "embulk/input/mixpanel_api/exceptions"
-require "range_generator"
-require "timezone_validator"
-require "active_support/core_ext/time"
+require "embulk/input/service/jql_service"
+require "embulk/input/service/export_service"
 
 module Embulk
   module Input
     class Mixpanel < InputPlugin
       Plugin.register_input("mixpanel", self)
-
-      NOT_PROPERTY_COLUMN = "event".freeze
-
-      # https://mixpanel.com/help/questions/articles/special-or-reserved-properties
-      # https://mixpanel.com/help/questions/articles/what-properties-do-mixpanels-libraries-store-by-default
-      #
-      # JavaScript to extract key names from HTML: run it on Chrome Devtool when opening their document
-      # > Array.from(document.querySelectorAll("strong")).map(function(s){ return s.textContent.match(/[A-Z]/) ? s.parentNode.textContent.match(/\((.*?)\)/)[1] : s.textContent.split(",").join(" ") }).join(" ")
-      # > Array.from(document.querySelectorAll("li")).map(function(s){ m = s.textContent.match(/\((.*?)\)/); return m && m[1] }).filter(function(k) { return k && !k.match("utm") }).join(" ")
-      KNOWN_KEYS = %W(
-        #{NOT_PROPERTY_COLUMN}
-        distinct_id ip mp_name_tag mp_note token time mp_country_code length campaign_id $email $phone $distinct_id $ios_devices $android_devices $first_name  $last_name  $name $city $region $country_code $timezone $unsubscribed
-        $city $region mp_country_code $browser $browser_version $device $current_url $initial_referrer $initial_referring_domain $os $referrer $referring_domain $screen_height $screen_width $search_engine $city $region $mp_country_code $timezone $browser_version $browser $initial_referrer $initial_referring_domain $os $last_seen $city $region mp_country_code $app_release $app_version $carrier $ios_ifa $os_version $manufacturer $lib_version $model $os $screen_height $screen_width $wifi $city $region $mp_country_code $timezone $ios_app_release $ios_app_version $ios_device_model $ios_lib_version $ios_version $ios_ifa $last_seen $city $region mp_country_code $app_version $bluetooth_enabled $bluetooth_version $brand $carrier $has_nfc $has_telephone $lib_version $manufacturer $model $os $os_version $screen_dpi $screen_height $screen_width $wifi $google_play_services $city $region mp_country_code $timezone $android_app_version $android_app_version_code $android_lib_version $android_os $android_os_version $android_brand $android_model $android_manufacturer $last_seen
-      ).uniq.freeze
-
-
-      DEFAULT_FETCH_DAYS = 7
-      DEFAULT_TIME_COLUMN = 'time'
-
+      
       def self.transaction(config, &control)
-        timezone = config.param(:timezone, :string)
-        TimezoneValidator.new(timezone).validate
-
-        from_date = config.param(:from_date, :string, default: (today(timezone) - 2).to_s)
-        fetch_days = config.param(:fetch_days, :integer, default: nil)
-
-
-        fetch_unknown_columns = config.param(:fetch_unknown_columns, :bool, default: false)
-
-        incremental_column = config.param(:incremental_column, :string, default: nil)
-        incremental = config.param(:incremental, :bool, default: true)
-        latest_fetched_time = config.param(:latest_fetched_time, :integer, default: 0)
-
-        # Backfill from date if incremental and an incremental field is set and we are in incremental run
-        if incremental && incremental_column && latest_fetched_time !=0
-          back_fill_days = config.param(:back_fill_days, :integer, default: 5)
-          Embulk.logger.info "Backfill days #{back_fill_days}"
-          from_date = (Date.parse(from_date) - back_fill_days).to_s
-          fetch_days = fetch_days.nil? ? nil : fetch_days + back_fill_days
-        end
-
-        range = RangeGenerator.new(from_date, fetch_days, timezone).generate_range
-        Embulk.logger.info "Try to fetch data from #{range.first} to #{range.last}"
-        job_start_time = Time.now.to_i*1000
-        upper_limit_delay = config.param(:incremental_column_upper_limit_delay_in_seconds, :integer, default: 0)
-        incremental_column_upper_limit = job_start_time - (upper_limit_delay * 1000)
-        task = {
-          params: export_params(config),
-          dates: range,
-          timezone: timezone,
-          export_endpoint: export_endpoint(config),
-          api_secret: config.param(:api_secret, :string),
-          schema: config.param(:columns, :array),
-          fetch_unknown_columns: fetch_unknown_columns,
-          fetch_custom_properties: config.param(:fetch_custom_properties, :bool, default: true),
-          retry_initial_wait_sec: config.param(:retry_initial_wait_sec, :integer, default: 1),
-          incremental_column: incremental_column,
-          retry_limit: config.param(:retry_limit, :integer, default: 5),
-          latest_fetched_time: latest_fetched_time,
-          incremental: incremental,
-          slice_range: config.param(:slice_range, :integer, default: 7),
-          job_start_time: job_start_time,
-          incremental_column_upper_limit: incremental_column_upper_limit,
-          allow_partial_import: config.param(:allow_partial_import,:bool, default: true)
-        }
-
-        if !incremental_column.nil? && !latest_fetched_time.nil? && (incremental_column_upper_limit <= latest_fetched_time)
-            raise Embulk::ConfigError.new("Incremental column upper limit (job_start_time - incremental_column_upper_limit_delay_in_seconds) can't be smaller or equal latest fetched time #{latest_fetched_time}")
-        end
-
-        if task[:fetch_unknown_columns] && task[:fetch_custom_properties]
-          raise Embulk::ConfigError.new("Don't set true both `fetch_unknown_columns` and `fetch_custom_properties`.")
-        end
+        service = service(config)
+        service.validate_config
+        task = service.create_task
+        Embulk.logger.info "Try to fetch data from #{task[:dates].first} to #{task[:dates].last}"
 
         columns = task[:schema].map do |column|
           name = column["name"]
@@ -91,15 +19,15 @@ module Embulk
           Column.new(nil, name, type, column["format"])
         end
 
-        if fetch_unknown_columns
-          Embulk.logger.warn "Deprecated `unknown_columns`. Use `fetch_custom_properties` instead."
-          columns << Column.new(nil, "unknown_columns", :json)
-        end
-
         if task[:fetch_custom_properties]
           columns << Column.new(nil, "custom_properties", :json)
         end
 
+        if task[:fetch_unknown_columns]
+          Embulk.logger.warn "Deprecated `unknown_columns`. Use `fetch_custom_properties` instead."
+          columns << Column.new(nil, "unknown_columns", :json)
+        end
+
         resume(task, columns, 1, &control)
       end
 
@@ -110,283 +38,37 @@ module Embulk
         # implementation is terrible.
         if task[:incremental]
           task_report = task_reports.first
-          next_to_date = Date.parse(task_report[:to_date])
-
-          next_config_diff = {
-            from_date: next_to_date.to_s,
-            latest_fetched_time: task_report[:latest_fetched_time],
-          }
-          return next_config_diff
+          service = service(task)
+          next_from_date = service.next_from_date(task_report)
+          return next_from_date
         end
         return {}
       end
 
       def self.guess(config)
-        giveup_when_mixpanel_is_down(export_endpoint(config))
-
-        retryer = perfect_retry({
-          retry_initial_wait_sec: config.param(:retry_initial_wait_sec, :integer, default: 1),
-          retry_limit: config.param(:retry_limit, :integer, default: 5),
-        })
-        client = MixpanelApi::Client.new(config.param(:api_secret, :string),
-                                         retryer,
-                                         export_endpoint(config))
-
-        range = guess_range(config)
-        Embulk.logger.info "Guessing schema using #{range.first}..#{range.last} records"
-
-        params = export_params(config).merge(
-          "from_date" => range.first,
-          "to_date" => range.last,
-        )
-        columns = guess_from_records(client.export_for_small_dataset(params))
-        return {"columns" => columns}
-      end
-
-      def self.perfect_retry(task)
-        PerfectRetry.new do |config|
-          config.limit = task[:retry_limit]
-          config.sleep = proc{|n| task[:retry_initial_wait_sec] * (2 * (n - 1)) }
-          config.dont_rescues = [Embulk::ConfigError,MixpanelApi::IncompleteExportResponseError]
-          config.rescues = [RuntimeError]
-          config.log_level = nil
-          config.logger = Embulk.logger
-        end
-      end
-
-      def self.export_endpoint(config)
-        config.param(:export_endpoint, :string, default: Embulk::Input::MixpanelApi::Client::DEFAULT_EXPORT_ENDPOINT)
+        service = service(config)
+        service.validate_config
+        return {"columns"=>service.guess_columns}
       end
 
       def init
-        @export_endpoint = task[:export_endpoint]
         @api_secret = task[:api_secret]
-        @params = task[:params]
-        @timezone = task[:timezone]
-        @schema = task[:schema]
-        @dates = task[:dates]
-        @fetch_unknown_columns = task[:fetch_unknown_columns]
-        @incremental_column = task[:incremental_column]
-        @incremental = task[:incremental]
       end
 
       def run
-        Embulk.logger.info "Job start time is #{task[:job_start_time]}"
-        self.class.giveup_when_mixpanel_is_down(task[:export_endpoint])
-        prev_latest_fetched_time = task[:latest_fetched_time] || 0
-        prev_latest_fetched_time_format = Time.at(prev_latest_fetched_time).strftime("%F %T %z")
-        current_latest_fetched_time = prev_latest_fetched_time
-        @dates.each_slice(task[:slice_range]) do |slice_dates|
-          ignored_fetched_record_count = 0
-          # There is the issue with Mixpanel time field during the transition from standard to daylight saving time
-          # in the US timezone i.e. 11 Mar 2018 2AM - 2:59AM, time within that period must not be existed,
-          # due to daylight saving, time will be forwarded 1 hour from 2AM to 3AM.
-          #
-          # All of records with wrong timezone will be ignored instead of throw exception out
-          ignored_wrong_daylight_tz_record_count = 0
-          unless preview?
-            Embulk.logger.info "Fetching data from #{slice_dates.first} to #{slice_dates.last} ..."
-          end
-          record_time_column=@incremental_column || DEFAULT_TIME_COLUMN
-          begin
-            fetch(slice_dates, prev_latest_fetched_time).each do |record|
-              if @incremental
-                if !record["properties"].include?(record_time_column)
-                  raise Embulk::ConfigError.new("Incremental column not exists in fetched data #{record_time_column}")
-                end
-                record_time = record["properties"][record_time_column]
-                if @incremental_column.nil?
-                  if record_time <= prev_latest_fetched_time
-                    ignored_fetched_record_count += 1
-                    next
-                  end
-                end
-
-                current_latest_fetched_time= [
-                  current_latest_fetched_time,
-                  record_time,
-                ].max
-              end
-              begin
-                values = extract_values(record)
-                if @fetch_unknown_columns
-                  unknown_values = extract_unknown_values(record)
-                  values << unknown_values.to_json
-                end
-                if task[:fetch_custom_properties]
-                  values << collect_custom_properties(record)
-                end
-                page_builder.add(values)
-              rescue TZInfo::PeriodNotFound
-                ignored_wrong_daylight_tz_record_count += 1
-              end
-            end
-          rescue MixpanelApi::IncompleteExportResponseError
-            if !task[:allow_partial_import]
-            #   re raise the exception if we don't allow partial import
-              raise
-            end
-          end
-          if ignored_fetched_record_count > 0
-            Embulk.logger.warn "Skipped already loaded #{ignored_fetched_record_count} records. These record times are older or equal than previous fetched record time (#{prev_latest_fetched_time} @ #{prev_latest_fetched_time_format})."
-          end
-          if ignored_wrong_daylight_tz_record_count > 0
-            Embulk.logger.warn "Skipped #{ignored_wrong_daylight_tz_record_count} records due to corrupted Mixpanel time transition from standard to daylight saving"
-          end
-          break if preview?
-        end
-        page_builder.finish
-        task_report = {
-          latest_fetched_time: current_latest_fetched_time,
-          to_date: @dates.last || today(@timezone) - 1,
-        }
-        task_report
+        Mixpanel::service(DataSource[task.to_a]).ingest(task, page_builder)
       end
 
       private
 
-      def self.giveup_when_mixpanel_is_down(export_endpoint)
-        unless MixpanelApi::Client.mixpanel_available?(export_endpoint)
-          raise Embulk::DataError.new("Mixpanel service is down. Please retry later.")
-        end
-      end
-
-      def extract_values(record)
-        @schema.map do |column|
-          extract_value(record, column["name"])
-        end
-      end
-
-      def extract_value(record, name)
-        case name
-        when NOT_PROPERTY_COLUMN
-          record[NOT_PROPERTY_COLUMN]
-        when "time"
-          time = record["properties"]["time"]
-          adjust_timezone(time)
+      def self.service(config)
+        jql_mode = config[:jql_mode]
+        if jql_mode
+          Service::JqlService.new(config)
         else
-          record["properties"][name]
-        end
-      end
-
-      def collect_custom_properties(record)
-        specified_columns = @schema.map{|col| col["name"]}
-        custom_keys = record["properties"].keys.find_all{|key| !KNOWN_KEYS.include?(key.to_s) && !specified_columns.include?(key.to_s) }
-        custom_keys.inject({}) do |result, key|
-          result.merge({
-            key => record["properties"][key]
-          })
+          Service::ExportService.new(config)
         end
       end
-
-      def extract_unknown_values(record)
-        record_keys = record["properties"].keys + [NOT_PROPERTY_COLUMN]
-        schema_keys = @schema.map {|column| column["name"]}
-        unknown_keys = record_keys - schema_keys
-
-        unless unknown_keys.empty?
-          Embulk.logger.warn("Unknown columns exists in record: #{unknown_keys.join(', ')}")
-        end
-
-        unknown_keys.inject({}) do |result, key|
-          result[key] = extract_value(record, key)
-          result
-        end
-      end
-
-      def fetch(dates, last_fetch_time, &block)
-        from_date = dates.first
-        to_date = dates.last
-        params = @params.merge(
-          "from_date" => from_date,
-          "to_date" => to_date
-        )
-        if !@incremental_column.nil? # can't do filter on time column, time column need to be filter manually.
-          params = params.merge(
-            "where" => "#{params['where'].nil? ? '' : "(#{params['where']}) and " }properties[\"#{@incremental_column}\"] > #{last_fetch_time || 0} and properties[\"#{@incremental_column}\"] < #{task[:incremental_column_upper_limit]}"
-          )
-        end
-        Embulk.logger.info "Where params is #{params["where"]}"
-        client = MixpanelApi::Client.new(@api_secret, self.class.perfect_retry(task), @export_endpoint)
-
-        if preview?
-          client.export_for_small_dataset(params)
-        else
-          Enumerator.new do |y|
-            client.export(params) do |record|
-              y << record
-            end
-          end
-        end
-      end
-
-      def adjust_timezone(epoch)
-        # Adjust timezone offset to get UTC time
-        # c.f. https://mixpanel.com/docs/api-documentation/exporting-raw-data-you-inserted-into-mixpanel#export
-        tz = TZInfo::Timezone.get(@timezone)
-        offset = tz.period_for_local(epoch, true).offset.utc_total_offset
-        epoch - offset
-      end
-
-      def preview?
-        begin
-          org.embulk.spi.Exec.isPreview()
-        rescue java.lang.NullPointerException => e
-          false
-        end
-      end
-
-      def self.export_params(config)
-        event = config.param(:event, :array, default: nil)
-        event = event.nil? ? nil : event.to_json
-        {
-          event: event,
-          where: config.param(:where, :string, default: nil),
-          bucket: config.param(:bucket, :string, default: nil),
-        }
-      end
-
-      def self.default_guess_start_date(timezone)
-        today(timezone) - DEFAULT_FETCH_DAYS - 1
-      end
-
-      def self.guess_range(config)
-        time_zone = config.param(:timezone, :string, default: "")
-        from_date = config.param(:from_date, :string, default: default_guess_start_date(time_zone).to_s)
-        fetch_days = config.param(:fetch_days, :integer, default: DEFAULT_FETCH_DAYS)
-        range = RangeGenerator.new(from_date, fetch_days, time_zone).generate_range
-        if range.empty?
-          return default_guess_start_date(time_zone)..(today(time_zone) - 1)
-        end
-        range
-      end
-
-      def self.guess_from_records(records)
-        sample_props = records.map {|r| r["properties"]}
-        schema = Guess::SchemaGuess.from_hash_records(sample_props)
-        columns = schema.map do |col|
-          next if col.name == "time"
-          result = {
-            name: col.name,
-            type: col.type,
-          }
-          result[:format] = col.format if col.format
-          result
-        end.compact
-        columns.unshift(name: NOT_PROPERTY_COLUMN, type: :string)
-        # Shift incremental column to top
-        columns.unshift(name: "time", type: :long)
-      end
-
-      def self.today(timezone)
-        if timezone.nil?
-          Date.today
-        else
-          zone = ActiveSupport::TimeZone[timezone]
-          zone.nil? ? Date.today : zone.today
-        end
-      end
-
     end
   end
 end