embulk-input-mixpanel 0.3.2 → 0.3.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: a9b930812eda7043923b65baf311b686f6a75955
-  data.tar.gz: 9e7c02c4232d7cfa94a0e2fe051d6cc18122a33b
+  metadata.gz: 48c99d14bae13fa0257e070bc5b99d444679392a
+  data.tar.gz: 8327dd45de51b2dab42754f212e5c6861621de7f
 SHA512:
-  metadata.gz: e77beb053f17adedbcb86a87004871721ed08bcca888e7d3b0754e673c3b0ec181231432145732846b005d6ebc84429ab5457aa4bbee5a9e403ce5984c915382
-  data.tar.gz: 8b23002ec143aef8aa686e67f2c58eb7eb2b62fd2d7bc05aa913b773eda93d845dc349543a5d0f9e010784049ca4de7bd58a3056ec8d63343498dbcd0ed4a743
+  metadata.gz: ecb93dd7bade9b667d94a5f5cea247d6f46b1e1ad87d6046f6377f9495456e40f91b97a44a1440cbd04b8d4bbb15a2a223fbed7612469e08e427e32bc4e94db9
+  data.tar.gz: 94e3728eed0ce17d7178557556077f1d37049881567715523955c9888560d4886d774440206570cbf463822f2f82f19b2d216aa293bd8be1d1f0b27d3ea566a1

data/CHANGELOG.md

@@ -1,3 +1,10 @@
+## 0.3.3 - 2015-10-29
+
+* [enhancement] Exponential backoff retry [#31](https://github.com/treasure-data/embulk-input-mixpanel/pull/31)
+* [enhancement] Treat unguessed columns [#30](https://github.com/treasure-data/embulk-input-mixpanel/pull/30)
+* [enhancement] Loosely guess [#27](https://github.com/treasure-data/embulk-input-mixpanel/pull/27)
+* [maintenance] Refactor [#26](https://github.com/treasure-data/embulk-input-mixpanel/pull/26)
+
 ## 0.3.2 - 2015-10-06
 
 * [enhancement] Support embulk 0.7 [#25](https://github.com/treasure-data/embulk-input-mixpanel/pull/25)

data/README.md

@@ -38,9 +38,13 @@ To get it, you should log in mixpanel website, and click gear icon at the lower
   - NOTE: Mixpanel API supports to export data from at least 2 days before to at most the previous day.
 - **fetch_days**: Count of days range for exporting (integer, optional, default: from_date - (today - 1))
   - NOTE: Mixpanel doesn't support to from_date > today - 2
+- **fetch_unknown_columns**: If you want this plugin fetches unknown (unconfigured in config) columns (boolean, optional, default: true)
+  - NOTE: If true, `unknown_columns` column is created and added unknown columns' data.
 - **event**: The event or events to filter data (array, optional, default: nil)
 - **where**: Expression to filter data (c.f. https://mixpanel.com/docs/api-documentation/data-export-api#segmentation-expressions) (string, optional, default: nil)
 - **bucket**:The data backet to filter data (string, optional, default: nil)
+- **retry_initial_wait_sec** Wait seconds for exponential backoff initial value (integer, default: 1)
+- **retry_limit**: Try to retry this times (integer, default: 5)
 
 ## Example
 

data/embulk-input-mixpanel.gemspec

@@ -1,7 +1,7 @@
 
 Gem::Specification.new do |spec|
   spec.name          = "embulk-input-mixpanel"
-  spec.version       = "0.3.2"
+  spec.version       = "0.3.3"
   spec.authors       = ["yoshihara", "uu59"]
   spec.summary       = "Mixpanel input plugin for Embulk"
   spec.description   = "Loads records from Mixpanel."

data/lib/embulk/input/mixpanel.rb

@@ -1,5 +1,7 @@
 require "tzinfo"
 require "embulk/input/mixpanel_api/client"
+require "range_generator"
+require "timezone_validator"
 
 module Embulk
   module Input
@@ -7,6 +9,7 @@ module Embulk
       Plugin.register_input("mixpanel", self)
 
       GUESS_RECORDS_COUNT = 10
+      NOT_PROPERTY_COLUMN = "event".freeze
 
       # NOTE: It takes long time to fetch data between from_date to
       # to_date by one API request. So this plugin fetches data
@@ -14,68 +17,36 @@ module Embulk
       SLICE_DAYS_COUNT = 7
 
       def self.transaction(config, &control)
-        task = {}
-
-        task[:params] = export_params(config)
-
-        begin
-          from_date_str = config.param(:from_date, :string, default: (Date.today - 2).to_s)
-          from_date = Date.parse(from_date_str)
-        rescue ArgumentError # invalid date
-          raise ConfigError.new "from_date '#{from_date_str}' is invalid date"
-        end
-
-        if from_date > Date.today - 1
-          Embulk.logger.warn "Mixpanel allow 2 days before to from_date, so no data is input."
-          target_dates = []
-        else
-          days = config.param(:fetch_days, :integer, default: nil)
-
-          if days.nil?
-            # When no 'days' is specified in config file, so dates is
-            # generated by from_date and yeasterday.
-            dates = from_date..(Date.today - 1)
-          elsif days < 1
-            raise ConfigError.new "days '#{days}' is invalid. Please specify bigger number than 0."
-          else
-            # When 'days' is specified in config file and it is satisfied,
-            # so it is used for dates.
-            dates = from_date..(from_date + days - 1)
-          end
-
-          target_dates = dates.find_all {|date| date < Date.today}
-
-          Embulk.logger.info "Try to fetch data from #{target_dates.first} to #{target_dates.last}"
-
-          overtimes = dates.to_a - target_dates
-          unless overtimes.empty?
-            Embulk.logger.warn "These dates are too early access, ignored them: from #{overtimes.first} to #{overtimes.last}"
-          end
-        end
-
-        task[:dates] = target_dates.map {|date| date.to_s}
-
-        task[:api_key] = config.param(:api_key, :string)
-        task[:api_secret] = config.param(:api_secret, :string)
-        task[:timezone] = config.param(:timezone, :string)
-
-        begin
-          # raises exception if timezone is invalid string
-          TZInfo::Timezone.get(task[:timezone])
-        rescue => e
-          Embulk.logger.error "'#{task[:timezone]}' is invalid timezone"
-          raise ConfigError.new e.message
-        end
+        timezone = config.param(:timezone, :string)
+        TimezoneValidator.new(timezone).validate
+
+        from_date = config.param(:from_date, :string, default: (Date.today - 2).to_s)
+        fetch_days = config.param(:fetch_days, :integer, default: nil)
+        range = RangeGenerator.new(from_date, fetch_days).generate_range
+        Embulk.logger.info "Try to fetch data from #{range.first} to #{range.last}"
+
+        task = {
+          params: export_params(config),
+          dates: range,
+          timezone: timezone,
+          api_key: config.param(:api_key, :string),
+          api_secret: config.param(:api_secret, :string),
+          schema: config.param(:columns, :array),
+          fetch_unknown_columns: config.param(:fetch_unknown_columns, :bool, default: true),
+          retry_initial_wait_sec: config.param(:retry_initial_wait_sec, :integer, default: 1),
+          retry_limit: config.param(:retry_limit, :integer, default: 5),
+        }
 
-        columns = []
-        task[:schema] = config.param(:columns, :array)
-        task[:schema].each do |column|
+        columns = task[:schema].map do |column|
           name = column["name"]
           type = column["type"].to_sym
 
-          columns << Column.new(nil, name, type, column["format"])
+          Column.new(nil, name, type, column["format"])
         end
 
+        # for unknown columns
+        columns << Column.new(nil, "unknown_columns", :string)
+
         resume(task, columns, 1, &control)
       end
 
@@ -94,36 +65,15 @@ module Embulk
       def self.guess(config)
         client = MixpanelApi::Client.new(config.param(:api_key, :string), config.param(:api_secret, :string))
 
-        from_date_str = config.param(:from_date, :string, default: (Date.today - 1 - SLICE_DAYS_COUNT).to_s)
-
-        from_date = Date.parse(from_date_str)
+        range = guess_range(config)
+        Embulk.logger.info "Guessing schema using #{range.first}..#{range.last} records"
 
-        if from_date > Date.today - 1
-          raise ConfigError.new "Please specify date later than yesterday (inclusive) as 'from_date'"
-        end
-
-        # NOTE: to_date is yeasterday if from_date..Date.Today doesn't have
-        # more SLICE_DAYS_COUNT days.
-        to_date = [from_date + SLICE_DAYS_COUNT, Date.today - 1].min
-
-        params = export_params(config)
-        params = params.merge(
-          from_date: from_date.to_s,
-          to_date: to_date.to_s,
+        params = export_params(config).merge(
+          from_date: range.first,
+          to_date: range.last,
         )
 
-        records = client.export(params)
-        sample_records = records.first(GUESS_RECORDS_COUNT)
-        properties = Guess::SchemaGuess.from_hash_records(sample_records.map{|r| r["properties"]})
-        columns = properties.map do |col|
-          result = {
-            name: col.name,
-            type: col.type,
-          }
-          result[:format] = col.format if col.format
-          result
-        end
-        columns.unshift(name: "event", type: :string)
+        columns = guess_from_records(client.export(params))
         return {"columns" => columns}
       end
 
@@ -134,33 +84,18 @@ module Embulk
         @timezone = task[:timezone]
         @schema = task[:schema]
         @dates = task[:dates]
+        @fetch_unknown_columns = task[:fetch_unknown_columns]
       end
 
       def run
-        client = MixpanelApi::Client.new(@api_key, @api_secret)
         @dates.each_slice(SLICE_DAYS_COUNT) do |dates|
-          from_date = dates.first
-          to_date = dates.last
-          Embulk.logger.info "Fetching data from #{from_date} to #{to_date} ..."
-
-          params = @params.merge(
-            "from_date" => from_date,
-            "to_date" => to_date
-          )
-
-          records = client.export(params)
-
-          records.each do |record|
-            values = @schema.map do |column|
-              case column["name"]
-              when "event"
-                record["event"]
-              when "time"
-                time = record["properties"]["time"]
-                adjust_timezone(time)
-              else
-                record["properties"][column["name"]]
-              end
+          Embulk.logger.info "Fetching data from #{dates.first} to #{dates.last} ..."
+
+          fetch(dates).each do |record|
+            values = extract_values(record)
+            if @fetch_unknown_columns
+              unknown_values = extract_unknown_values(record)
+              values << unknown_values.to_json
             end
             page_builder.add(values)
           end
@@ -176,6 +111,50 @@ module Embulk
 
       private
 
+      def extract_values(record)
+        @schema.map do |column|
+          extract_value(record, column["name"])
+        end
+      end
+
+      def extract_value(record, name)
+        case name
+        when NOT_PROPERTY_COLUMN
+          record[NOT_PROPERTY_COLUMN]
+        when "time"
+          time = record["properties"]["time"]
+          adjust_timezone(time)
+        else
+          record["properties"][name]
+        end
+      end
+
+      def extract_unknown_values(record)
+        record_keys = record["properties"].keys + [NOT_PROPERTY_COLUMN]
+        schema_keys = @schema.map {|column| column["name"]}
+        unknown_keys = record_keys - schema_keys
+
+        unless unknown_keys.empty?
+          Embulk.logger.warn("Unknown columns exists in record: #{unknown_keys.join(', ')}")
+        end
+
+        unknown_keys.inject({}) do |result, key|
+          result[key] = extract_value(record, key)
+          result
+        end
+      end
+
+      def fetch(dates)
+        from_date = dates.first
+        to_date = dates.last
+        params = @params.merge(
+          "from_date" => from_date,
+          "to_date" => to_date,
+        )
+        client = MixpanelApi::Client.new(@api_key, @api_secret)
+        client.export_with_retry(params, task[:retry_initial_wait_sec], task[:retry_limit])
+      end
+
       def adjust_timezone(epoch)
         # Adjust timezone offset to get UTC time
         # c.f. https://mixpanel.com/docs/api-documentation/exporting-raw-data-you-inserted-into-mixpanel#export
@@ -203,6 +182,34 @@ module Embulk
           bucket: config.param(:bucket, :string, default: nil),
         }
       end
+
+      def self.default_guess_start_date
+        Date.today - SLICE_DAYS_COUNT - 1
+      end
+
+      def self.guess_range(config)
+        from_date = config.param(:from_date, :string, default: default_guess_start_date.to_s)
+        fetch_days = config.param(:fetch_days, :integer, default: SLICE_DAYS_COUNT)
+        range = RangeGenerator.new(from_date, fetch_days).generate_range
+        if range.empty?
+          return default_guess_start_date..(Date.today - 1)
+        end
+        range
+      end
+
+      def self.guess_from_records(records)
+        sample_props = records.first(GUESS_RECORDS_COUNT).map{|r| r["properties"]}
+        schema = Guess::SchemaGuess.from_hash_records(sample_props)
+        columns = schema.map do |col|
+          result = {
+            name: col.name,
+            type: col.type,
+          }
+          result[:format] = col.format if col.format
+          result
+        end
+        columns.unshift(name: NOT_PROPERTY_COLUMN, type: :string)
+      end
     end
 
   end

data/lib/embulk/input/mixpanel_api/client.rb

@@ -15,32 +15,68 @@ module Embulk
           @api_secret = api_secret
         end
 
+        def export_with_retry(params = {}, retry_initial_wait_sec, retry_limit)
+          body = with_retry(retry_initial_wait_sec, retry_limit) do
+            request(params)
+          end
+
+          response_to_enum(body)
+        end
+
         def export(params = {})
+          body = request(params)
+          response_to_enum(body)
+        end
+
+        private
+
+        def response_to_enum(response_body)
+          Enumerator.new do |y|
+            response_body.lines.each do |json|
+              # TODO: raise Embulk::DataError when invalid json given for Embulk 0.7+
+              y << JSON.parse(json)
+            end
+          end
+        end
+
+        def request(params)
           # https://mixpanel.com/docs/api-documentation/exporting-raw-data-you-inserted-into-mixpanel
           params[:expire] ||= Time.now.to_i + TIMEOUT_SECONDS
           params[:sig] = signature(params)
-
           Embulk.logger.debug "Export param: #{params.to_s}"
 
           response = httpclient.get(ENDPOINT_EXPORT, params)
-
           Embulk.logger.debug "response code: #{response.code}"
-
-          if (400..499).include?(response.code)
+          case response.code
+          when 400..499
             raise ConfigError.new response.body
-          elsif response.code >= 500
+          when 500..599
             raise RuntimeError, response.body
           end
+          response.body
+        end
 
-          Enumerator.new do |y|
-            response.body.lines.each do |json|
-              y << JSON.parse(json)
+        def with_retry(initial_wait, retry_limit, &block)
+          retry_count = 0
+          wait_sec = initial_wait
+          begin
+            yield
+          rescue Embulk::ConfigError => e # TODO: rescue Embulk::DataError for Embulk 0.7+
+            # Don't retry
+            raise e
+          rescue => e
+            if retry_limit <= retry_count
+              Embulk.logger.error "'#{e}(#{e.class})' error occured and reached retry limit (#{retry_limit} times)"
+              raise e
             end
+            retry_count += 1
+            Embulk.logger.warn "Retrying after #{wait_sec} seconds [#{retry_count}/#{retry_limit}] '#{e}(#{e.class})' error occured"
+            sleep wait_sec
+            wait_sec *= 2
+            retry
           end
         end
 
-        private
-
         def signature(params)
           # https://mixpanel.com/docs/api-documentation/data-export-api#auth-implementation
           sorted_keys = params.keys.map(&:to_s).sort

data/lib/range_generator.rb

@@ -0,0 +1,79 @@
+class RangeGenerator
+  attr_reader :from_date_str, :fetch_days
+
+  def initialize(from_date_str, fetch_days)
+    @from_date_str = from_date_str
+    @fetch_days = fetch_days
+  end
+
+  def generate_range
+    validate
+    show_warnings
+    range_only_past.map{|date| date.to_s}
+  end
+
+  private
+
+  def from_date
+    Date.parse(from_date_str)
+  end
+
+  def validate
+    begin
+      from_date
+    rescue ArgumentError # invalid date
+      raise Embulk::ConfigError.new "from_date '#{from_date_str}' is invalid date"
+    end
+
+    if fetch_days && fetch_days < 1
+      # `days` only allowed nil or positive number
+      raise Embulk::ConfigError.new "fetch_days '#{fetch_days}' is invalid. Please specify bigger number than 0."
+    end
+  end
+
+  def show_warnings
+    if from_date_too_early?
+      Embulk.logger.warn "Mixpanel allow 2 days before to from_date, so no data is input."
+    end
+
+    if overdays?
+      Embulk.logger.warn "These dates are too early access, ignored them: from #{overdays.first} to #{overdays.last}"
+    end
+  end
+
+  def range
+    if from_date_too_early?
+      return []
+    end
+
+    if fetch_days
+      from_date..(from_date + fetch_days - 1)
+    else
+      from_date..yesterday
+    end
+  end
+
+  def range_only_past
+    range.find_all{|date| date < today}
+  end
+
+  def overdays?
+    ! overdays.empty?
+  end
+
+  def overdays
+    range.to_a - range_only_past.to_a
+  end
+
+  def from_date_too_early?
+    from_date > yesterday
+  end
+
+  def yesterday
+    today - 1
+  end
+
+  def today
+    @today ||= Date.today
+  end
+end

data/lib/timezone_validator.rb

@@ -0,0 +1,15 @@
+class TimezoneValidator
+  def initialize(timezone)
+    @timezone = timezone
+  end
+
+  def validate
+    begin
+      # raises exception if timezone is invalid string
+      TZInfo::Timezone.get(@timezone)
+    rescue => e
+      Embulk.logger.error "'#{@timezone}' is invalid timezone"
+      raise Embulk::ConfigError.new e.message
+    end
+  end
+end

data/test/embulk/input/mixpanel_api/test_client.rb

@@ -89,6 +89,71 @@ module Embulk
             end
           end
 
+          class ExportRetryTest < self
+            def setup
+              @httpclient = HTTPClient.new
+              @client = Client.new(API_KEY, API_SECRET)
+              @retry_initial_wait_sec = 1
+              @retry_limit = 3
+              stub_client
+            end
+
+            def test_retry_with_500
+              stub_response(failure_response(500))
+
+              @retry_limit.times do |n|
+                mock(@client).sleep(@retry_initial_wait_sec * (2**n))
+              end
+              mock(Embulk.logger).warn(/retry/i).times(@retry_limit)
+              mock(Embulk.logger).error(/retry/i).once
+
+              assert_raise do
+                @client.export_with_retry(params, @retry_initial_wait_sec, @retry_limit)
+              end
+            end
+
+            def test_retry_with_timeout
+              @httpclient.connect_timeout = 0.000000000000000000001
+
+              @retry_limit.times do |n|
+                mock(@client).sleep(@retry_initial_wait_sec * (2**n))
+              end
+              mock(Embulk.logger).warn(/retry/i).times(@retry_limit)
+              mock(Embulk.logger).error(/retry/i).once
+
+              assert_raise(HTTPClient::TimeoutError) do
+                @client.export_with_retry(params, @retry_initial_wait_sec, @retry_limit)
+              end
+            end
+
+            def test_not_retry_with_401
+              @httpclient.test_loopback_http_response << "HTTP/1.1 401\r\n\r\n"
+              mock(Embulk.logger).warn(/retry/i).never
+              mock(Embulk.logger).error(/retry/i).never
+
+              assert_raise(Embulk::ConfigError) do
+                @client.export_with_retry(params, 0, 1).each do |record|
+                  record
+                end
+              end
+            end
+
+            def test_not_retry_with_invalid_json
+              omit "Embulk 0.6 or earlier has no DataError (enable to uncomment below code)"
+
+              # @httpclient.test_loopback_http_response << "HTTP/1.1 200\r\n\r\ninvalid json"
+              # mock(Embulk.logger).warn(/retry/i).never
+              # mock(Embulk.logger).error(/retry/i).never
+
+              # assert_raise(Embulk::DataError) do
+              #   @client.export_with_retry(params, 0, 1).each do |record|
+              #     # DataError will raised in each block
+              #     record
+              #   end
+              # end
+            end
+          end
+
           private
 
           def stub_client

data/test/embulk/input/test_mixpanel.rb

@@ -50,6 +50,11 @@ module Embulk
       end
 
       class GuessTest < self
+        def setup
+          # Do nothing from parent
+          mute_warn
+        end
+
         def test_from_date_old_date
           config = {
             type: "mixpanel",
@@ -58,9 +63,8 @@ module Embulk
             from_date: FROM_DATE,
           }
 
-          from_date = config[:from_date]
-          to_date = Date.parse(from_date) + Mixpanel::SLICE_DAYS_COUNT
-          stub_export(from_date, to_date)
+          stub_export_all
+          mock(Embulk.logger).info(/^Guessing.*#{Regexp.escape FROM_DATE}\.\./)
 
           actual = Mixpanel.guess(embulk_config(config))
           assert_equal(expected, actual)
@@ -74,25 +78,25 @@ module Embulk
             from_date: Date.today.to_s,
           }
 
-          assert_raise(ConfigError) do
-            Mixpanel.guess(embulk_config(config))
-          end
+          stub_export_all
+          mock(Embulk.logger).info(/Guessing.*#{Regexp.escape Mixpanel.default_guess_start_date.to_s}/)
+
+          Mixpanel.guess(embulk_config(config))
         end
 
         def test_from_date_yesterday
+          from_date = (Date.today - 1).to_s
           config = {
             type: "mixpanel",
             api_key: API_KEY,
             api_secret: API_SECRET,
-            from_date: (Date.today - 1).to_s,
+            from_date: from_date,
           }
 
-          from_date = config[:from_date]
-          to_date = from_date
-          stub_export(from_date, to_date)
+          stub_export_all
+          mock(Embulk.logger).info(/Guessing.*#{Regexp.escape from_date}/)
 
-          actual = Mixpanel.guess(embulk_config(config))
-          assert_equal(expected, actual)
+          Mixpanel.guess(embulk_config(config))
         end
 
         def test_no_from_date
@@ -102,31 +106,24 @@ module Embulk
             api_secret: API_SECRET,
           }
 
-          from_date = Date.today - 1 - Mixpanel::SLICE_DAYS_COUNT
-          to_date = Date.today - 1
-          stub_export(from_date, to_date)
+          stub_export_all
+          mock(Embulk.logger).info(/Guessing.*#{Regexp.escape Mixpanel.default_guess_start_date.to_s}/)
 
-          actual = Mixpanel.guess(embulk_config(config))
-          assert_equal(expected, actual)
+          Mixpanel.guess(embulk_config(config))
         end
 
         private
 
-        def stub_export(from_date, to_date)
-          params = {
-            api_key: API_KEY,
-            event: nil,
-            where: nil,
-            bucket: nil,
-            from_date: from_date.to_s,
-            to_date: to_date.to_s,
-          }
-
+        def stub_export_all
           any_instance_of(MixpanelApi::Client) do |klass|
-            stub(klass).export(params) { records }
+            stub(klass).export(anything) { records }
           end
         end
 
+        def mute_warn
+          stub(Embulk.logger).warn(anything) {}
+        end
+
         def embulk_config(config)
           DataSource[*config.to_a.flatten(1)]
         end
@@ -336,9 +333,12 @@ module Embulk
         end
 
         def columns
-          schema.map do |col|
+          unknown_columns = [Column.new(nil, "unknown_columns", :string)]
+          configured_columns = schema.map do |col|
             Column.new(nil, col["name"], col["type"].to_sym)
           end
+
+          configured_columns + unknown_columns
         end
       end
 
@@ -371,7 +371,7 @@ module Embulk
         def setup_client
 
           any_instance_of(MixpanelApi::Client) do |klass|
-            stub(klass).export(anything) { records }
+            stub(klass).request { records_raw_response }
           end
         end
 
@@ -415,6 +415,52 @@ module Embulk
           @plugin.run
         end
 
+        class UnknownColumnsTest < self
+          def setup
+            super
+            @page_builder = Object.new
+            @plugin = Mixpanel.new(task, nil, nil, @page_builder)
+          end
+
+          def test_run
+            Embulk.logger.warn(anything)
+            stub(@plugin).preview? { false }
+
+            # NOTE: Expect records are contained same record
+            record = records.first
+            properties = record["properties"]
+
+            time = properties["time"]
+            tz = TZInfo::Timezone.get(TIMEZONE)
+            offset = tz.period_for_local(time, true).offset.utc_offset
+            adjusted_time = time - offset
+
+            added = [
+              properties["foo"],
+              adjusted_time,
+              {"int" => properties["int"], "event" => record["event"]}.to_json
+            ]
+
+            mock(@page_builder).add(added).times(records.length * 2)
+            mock(@page_builder).finish
+
+            @plugin.run
+          end
+
+          private
+
+          def task
+            super.merge(schema: schema, fetch_unknown_columns: true)
+          end
+
+          def schema
+            [
+              {"name" => "foo", "type" => "long"},
+              {"name" => "time", "type" => "long"},
+            ]
+          end
+        end
+
         private
 
         def timezone_offset_seconds
@@ -440,6 +486,9 @@ module Embulk
           schema: schema,
           dates: DATES.to_a.map(&:to_s),
           params: Mixpanel.export_params(embulk_config),
+          fetch_unknown_columns: false,
+          retry_initial_wait_sec: 2,
+          retry_limit: 3,
         }
       end
 
@@ -456,6 +505,10 @@ module Embulk
         ] * 30
       end
 
+      def records_raw_response
+        records.map(&:to_json).join("\n")
+      end
+
       def record_epoch
         1234567890
       end
@@ -467,6 +520,9 @@ module Embulk
           api_secret: API_SECRET,
           from_date: FROM_DATE,
           fetch_days: DAYS,
+          fetch_unknown_columns: false,
+          retry_initial_wait_sec: 2,
+          retry_limit: 3,
         }
       end
 

data/test/test_range_generator.rb

@@ -0,0 +1,94 @@
+require "range_generator"
+require "override_assert_raise"
+
+class RangeGeneratorTest < Test::Unit::TestCase
+  include OverrideAssertRaise
+
+  class GenerateRangeTest < self
+    data do
+      {
+        from_date: ["aaaaaaaaa", 1],
+        fetch_days: ["2010-01-01", -9],
+      }
+    end
+    def test_invalid(args)
+      assert_raise(Embulk::ConfigError) do
+        generate_range(*args)
+      end
+    end
+
+    def test_all_days_past
+      days = 5
+      from = "2010-01-01"
+      expected_from = Date.parse(from)
+      expected_to = Date.parse("2010-01-05")
+
+      expected = (expected_from..expected_to).to_a.map{|date| date.to_s}
+
+      actual = RangeGenerator.new(from, days).generate_range
+
+      assert_equal(expected, actual)
+    end
+
+    class OverDaysTest < self
+      def setup
+        @from = Date.today - 5
+        @days = 10
+        @warn_message_regexp = /ignored them/
+      end
+
+      def test_range_only_past
+        expected_to = Date.today - 1
+        expected = (@from..expected_to).to_a.map{|date| date.to_s}
+
+        stub(Embulk.logger).warn(@warn_message_regexp)
+
+        assert_equal(expected, generate_range)
+      end
+
+      def test_warn
+        mock(Embulk.logger).warn(@warn_message_regexp)
+
+        generate_range
+      end
+
+      private
+
+      def generate_range
+        super(@from.to_s, @days)
+      end
+    end
+
+    class FromDateEarlyTest < self
+      def setup
+        @from = Date.today + 5
+        @days = 10
+        @warn_message_regexp = /allow 2 days/
+      end
+
+      def test_empty_range
+        stub(Embulk.logger).warn(@warn_message_regexp)
+
+        assert_equal([], generate_range)
+      end
+
+      def test_warn
+        mock(Embulk.logger).warn(@warn_message_regexp)
+
+        generate_range
+      end
+
+      private
+
+      def generate_range
+        super(@from.to_s, @days)
+      end
+    end
+
+    private
+
+    def generate_range(from_date_str, fetch_days)
+      RangeGenerator.new(from_date_str, fetch_days).generate_range
+    end
+  end
+end

data/test/test_timezone_validator.rb

@@ -0,0 +1,24 @@
+require "timezone_validator"
+require "override_assert_raise"
+
+class TimezoneValidatorTest < Test::Unit::TestCase
+  include OverrideAssertRaise
+
+  def test_valid
+    valid_timezone = "Asia/Tokyo"
+
+    assert_nothing_raised do
+      TimezoneValidator.new(valid_timezone).validate
+    end
+  end
+
+  def test_invalid
+    invalid_timezone = "Asia/Tokyoooooooooooooo"
+
+    mock(Embulk.logger).error(/#{Regexp.new(invalid_timezone)}/)
+
+    assert_raise(Embulk::ConfigError) do
+      TimezoneValidator.new(invalid_timezone).validate
+    end
+  end
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: embulk-input-mixpanel
 version: !ruby/object:Gem::Version
-  version: 0.3.2
+  version: 0.3.3
 platform: ruby
 authors:
 - yoshihara
@@ -9,7 +9,7 @@ authors:
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2015-10-06 00:00:00.000000000 Z
+date: 2015-10-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -199,11 +199,15 @@ files:
 - gemfiles/template.erb
 - lib/embulk/input/mixpanel.rb
 - lib/embulk/input/mixpanel_api/client.rb
+- lib/range_generator.rb
+- lib/timezone_validator.rb
 - test/embulk/input/mixpanel_api/test_client.rb
 - test/embulk/input/test_mixpanel.rb
 - test/override_assert_raise.rb
 - test/prepare_embulk.rb
 - test/run-test.rb
+- test/test_range_generator.rb
+- test/test_timezone_validator.rb
 homepage: https://github.com/treasure-data/embulk-input-mixpanel
 licenses:
 - Apache2
@@ -234,3 +238,5 @@ test_files:
 - test/override_assert_raise.rb
 - test/prepare_embulk.rb
 - test/run-test.rb
+- test/test_range_generator.rb
+- test/test_timezone_validator.rb