embulk-input-mixpanel 0.2.0 → 0.2.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 5fca90578e059bd058fa52d1d8283afbd85a58a1
-  data.tar.gz: 09305de32323e4284f33a005330d522f2afc98e7
+  metadata.gz: 9a688fb2eb175f13ee5c82b8ea2da6c74ed69839
+  data.tar.gz: af0fdae8cc6fba0911ab269d7d8f354d51803f25
 SHA512:
-  metadata.gz: 8a200ccc100c2baa7552e933846648253bace0b944a7cefa8bfcc87198ab0e5dfe2a8a754a3195ac2aeca096114a92defc74c3d7e35176a2d8475bcac830cf83
-  data.tar.gz: 91c0591f7bf83a14dc5bdb7f56f2915c97ba8e3d3b81cac8f37ac411541c0ae5302bf2712c6f4e6347a542955c5278a085cab67925a8da6559ce0195dc12edaf
+  metadata.gz: 25381a19a9743a0b945d5d453931b1f93ab87009a79185b72eba32577a496550f1050b38ff8baa81c164d628a1c67714e785c27bf6833a04e69431bcad2e6ace
+  data.tar.gz: d69d65730c8c3a3d1ea1c85182e274ea8bde716662c132ee628e1c0be7d25e2b97c0a2c15dc0a7269b714677cf1635f04fcdd8a8237ec7353714d4f1ede05094

data/.travis.yml

@@ -23,3 +23,4 @@ gemfile:
 matrix:
   allow_failures:
     - gemfile: gemfiles/embulk-0.6.22
+    - gemfile: gemfiles/embulk-latest # embulk 0.7.1

data/CHANGELOG.md

@@ -1,3 +1,8 @@
+## 0.2.1 - 2015-08-26
+
+* [fixed] Fix guess with recently from date [#18](https://github.com/treasure-data/embulk-input-mixpanel/pull/18)
+* [fixed] Fix error handling when invalid date set given [#17](https://github.com/treasure-data/embulk-input-mixpanel/pull/17)
+
 ## 0.2.0 - 2015-08-17
 
 * [enhanement] Raise config error for unretryable [#15](https://github.com/treasure-data/embulk-input-mixpanel/pull/15) [[Reported by @muga](https://github.com/treasure-data/embulk-input-mixpanel/issues/11). Thanks!!]

data/README.md

@@ -35,7 +35,9 @@ To get it, you should log in mixpanel website, and click gear icon at the lower
 - **api_secret**: project API Secret (string, required)
 - **timezone**: project timezone(string, required)
 - **from_date**: From date to export (string, optional, default: today - 2)
-- **days**: Count of days range for exporting (integer, optional, default: (today - 1) - from_date)
+  - NOTE: Mixpanel API supports to export data from at least 2 days before to at most the previous day.
+- **days**: Count of days range for exporting (integer, optional, default: from_date - (today - 1))
+  - NOTE: Mixpanel doesn't support to from_date > today - 2
 - **event**: The event or events to filter data (array, optional, default: nil)
 - **where**: Expression to filter data (c.f. https://mixpanel.com/docs/api-documentation/data-export-api#segmentation-expressions) (string, optional, default: nil)
 - **bucket**:The data backet to filter data (string, optional, default: nil)

data/embulk-input-mixpanel.gemspec

@@ -1,7 +1,7 @@
 
 Gem::Specification.new do |spec|
   spec.name          = "embulk-input-mixpanel"
-  spec.version       = "0.2.0"
+  spec.version       = "0.2.1"
   spec.authors       = ["yoshihara", "uu59"]
   spec.summary       = "Mixpanel input plugin for Embulk"
   spec.description   = "Loads records from Mixpanel."

data/lib/embulk/input/mixpanel.rb

@@ -18,8 +18,40 @@ module Embulk
 
         task[:params] = export_params(config)
 
-        dates = generate_dates(config)
-        task[:dates] = dates.map {|date| date.to_s}
+        begin
+          from_date_str = config.param(:from_date, :string, default: (Date.today - 2).to_s)
+          from_date = Date.parse(from_date_str)
+        rescue ArgumentError # invalid date
+          raise ConfigError, "from_date '#{from_date_str}' is invalid date"
+        end
+
+        if from_date > Date.today - 1
+          Embulk.logger.warn "Mixpanel allow 2 days before to from_date, so no data is input."
+          target_dates = []
+        else
+          days = config.param(:days, :integer, default: nil)
+
+          if days.nil?
+            # When no 'days' is specified in config file, so dates is
+            # generated by from_date and yeasterday.
+            dates = from_date..(Date.today - 1)
+          elsif days < 1
+            raise ConfigError, "days '#{days}' is invalid. Please specify bigger number than 0."
+          else
+            # When 'days' is specified in config file and it is satisfied,
+            # so it is used for dates.
+            dates = from_date..(from_date + days)
+          end
+
+          target_dates = dates.find_all {|date| date < Date.today}
+
+          overtimes = dates.to_a - target_dates
+          unless overtimes.empty?
+            Embulk.logger.warn "These dates are too early access, ignored them: #{overtimes.map(&:to_s).join(', ')}"
+          end
+        end
+
+        task[:dates] = target_dates.map {|date| date.to_s}
 
         task[:api_key] = config.param(:api_key, :string)
         task[:api_secret] = config.param(:api_secret, :string)
@@ -60,14 +92,22 @@ module Embulk
       def self.guess(config)
         client = MixpanelApi::Client.new(config.param(:api_key, :string), config.param(:api_secret, :string))
 
-        from_date = config.param(:from_date, :string)
-        # NOTE: It should have 7 days beteen from_date and to_date
-        to_date = (Date.parse(from_date) + SLICE_DAYS_COUNT - 1).to_s
+        from_date_str = config.param(:from_date, :string, default: (Date.today - 1 - SLICE_DAYS_COUNT).to_s)
+
+        from_date = Date.parse(from_date_str)
+
+        if from_date > Date.today - 1
+          raise ConfigError, "Please specify date later than yesterday (inclusive) as 'from_date'"
+        end
+
+        # NOTE: to_date is yeasterday if from_date..Date.Today doesn't have
+        # more SLICE_DAYS_COUNT days.
+        to_date = [from_date + SLICE_DAYS_COUNT, Date.today - 1].min
 
         params = export_params(config)
         params = params.merge(
-          from_date: from_date,
-          to_date: to_date,
+          from_date: from_date.to_s,
+          to_date: to_date.to_s,
         )
 
         records = client.export(params)
@@ -85,25 +125,6 @@ module Embulk
         return {"columns" => columns}
       end
 
-      def self.generate_dates(config)
-        default_from_date = (Date.today - 2).to_s
-
-        begin
-          from_date = Date.parse(config.param(:from_date, :string, default: default_from_date))
-        rescue ArgumentError # invalid date
-          raise ConfigError, "from_date '#{from_date}' is invalid date"
-        end
-
-        default_days = ((Date.today - 1) - from_date).to_i
-        days = config.param(:days, :integer, default: default_days)
-
-        if days < 1
-          raise ConfigError, "days '#{days}' is invalid. Please spcify bigger number than 0."
-        end
-
-        from_date..(from_date + days)
-      end
-
       def init
         @api_key = task[:api_key]
         @api_secret = task[:api_secret]
@@ -147,7 +168,7 @@ module Embulk
 
         page_builder.finish
 
-        commit_report = {to_date: @dates.last}
+        commit_report = {to_date: @dates.last || (Date.today - 1)}
         return commit_report
       end
 

data/lib/embulk/input/mixpanel_api/client.rb

@@ -19,8 +19,13 @@ module Embulk
           # https://mixpanel.com/docs/api-documentation/exporting-raw-data-you-inserted-into-mixpanel
           params[:expire] ||= Time.now.to_i + TIMEOUT_SECONDS
           params[:sig] = signature(params)
+
+          Embulk.logger.debug "Export param: #{params.to_s}"
+
           response = httpclient.get(ENDPOINT_EXPORT, params)
 
+          Embulk.logger.debug "response code: #{response.code}"
+
           if (400..499).include?(response.code)
             raise ConfigError, response.body
           elsif response.code >= 500

data/test/embulk/input/mixpanel_api/test_client.rb

@@ -9,6 +9,7 @@ module Embulk
 
         def setup
           @client = Client.new(API_KEY, API_SECRET)
+          stub(Embulk).logger { ::Logger.new(IO::NULL) }
         end
 
         # NOTE: Client#signature is private method but this value

data/test/embulk/input/test_mixpanel.rb

@@ -36,6 +36,7 @@ module Embulk
             from_date = duration[:from_date]
             to_date = duration[:to_date]
 
+            params = params.merge(from_date: from_date, to_date: to_date)
             stub(klass).export(params) { records }
           end
         end
@@ -45,35 +46,126 @@ module Embulk
         stub(Embulk).logger { ::Logger.new(IO::NULL) }
       end
 
-      def test_guess
-        expected = {
-          "columns" => [
-            {name: "event", type: :string},
-            {name: "foo", type: :string},
-            {name: "time", type: :long},
-            {name: "int", type: :long},
-          ]
-        }
+      class GuessTest < self
+        def test_from_date_old_date
+          config = {
+            type: "mixpanel",
+            api_key: API_KEY,
+            api_secret: API_SECRET,
+            from_date: FROM_DATE,
+          }
 
-        actual = Mixpanel.guess(embulk_config)
-        assert_equal(expected, actual)
+          from_date = config[:from_date]
+          to_date = Date.parse(from_date) + Mixpanel::SLICE_DAYS_COUNT
+          stub_export(from_date, to_date)
+
+          actual = Mixpanel.guess(embulk_config(config))
+          assert_equal(expected, actual)
+        end
+
+        def test_from_date_today
+          config = {
+            type: "mixpanel",
+            api_key: API_KEY,
+            api_secret: API_SECRET,
+            from_date: Date.today.to_s,
+          }
+
+          assert_raise(ConfigError) do
+            Mixpanel.guess(embulk_config(config))
+          end
+        end
+
+        def test_from_date_yesterday
+          config = {
+            type: "mixpanel",
+            api_key: API_KEY,
+            api_secret: API_SECRET,
+            from_date: (Date.today - 1).to_s,
+          }
+
+          from_date = config[:from_date]
+          to_date = from_date
+          stub_export(from_date, to_date)
+
+          actual = Mixpanel.guess(embulk_config(config))
+          assert_equal(expected, actual)
+        end
+
+        def test_no_from_date
+          config = {
+            type: "mixpanel",
+            api_key: API_KEY,
+            api_secret: API_SECRET,
+          }
+
+          from_date = Date.today - 1 - Mixpanel::SLICE_DAYS_COUNT
+          to_date = Date.today - 1
+          stub_export(from_date, to_date)
+
+          actual = Mixpanel.guess(embulk_config(config))
+          assert_equal(expected, actual)
+        end
+
+        private
+
+        def stub_export(from_date, to_date)
+          params = {
+            api_key: API_KEY,
+            event: nil,
+            where: nil,
+            bucket: nil,
+            from_date: from_date.to_s,
+            to_date: to_date.to_s,
+          }
+
+          any_instance_of(MixpanelApi::Client) do |klass|
+            stub(klass).export(params) { records }
+          end
+        end
+
+        def embulk_config(config)
+          DataSource[*config.to_a.flatten(1)]
+        end
+
+        def expected
+          {
+            "columns" => [
+              {name: "event", type: :string},
+              {name: "foo", type: :string},
+              {name: "time", type: :long},
+              {name: "int", type: :long},
+            ]
+          }
+        end
       end
 
-      class GenerateDatesTest < self
+      class TransactionDateTest < self
         def test_valid_from_date
           from_date = "2015-08-14"
-          parsed_date = Date.parse(from_date)
+          mock(Mixpanel).resume(anything, anything, 1)
 
-          expected = (parsed_date..(parsed_date + DAYS))
-          actual = Mixpanel.generate_dates(transaction_config(from_date))
-          assert_equal(expected, actual)
+          Mixpanel.transaction(transaction_config(from_date))
         end
 
         def test_invalid_from_date
           from_date = "2015-08-41"
 
           assert_raise(Embulk::ConfigError) do
-            Mixpanel.generate_dates(transaction_config(from_date))
+            Mixpanel.transaction(transaction_config(from_date))
+          end
+        end
+
+        def test_future
+          from_date = (Date.today + 10).to_s
+          mock(Mixpanel).resume(anything, anything, 1)
+
+          Mixpanel.transaction(transaction_config(from_date))
+        end
+
+        def test_negative_days
+          assert_raise(Embulk::ConfigError) do
+            Mixpanel.transaction(transaction_config((Date.today - 1).to_s).merge(days: -1))
           end
         end
 
@@ -91,6 +183,42 @@ module Embulk
 
       class TransactionTest < self
         class FromDateTest < self
+          def setup
+          end
+
+          def test_ignore_early_days
+            stub(Embulk).logger { Logger.new(File::NULL) }
+
+            mock(Mixpanel).resume(task.merge(dates: target_dates), columns, 1, &control)
+            Mixpanel.transaction(transaction_config, &control)
+          end
+
+          def test_warn
+            stub(Mixpanel).resume(task.merge(dates: target_dates), columns, 1, &control)
+            mock(Embulk.logger).warn(anything)
+
+            Mixpanel.transaction(transaction_config, &control)
+          end
+
+          private
+
+          def dates
+            (Date.today - 10)..(Date.today + 10)
+          end
+
+          def target_dates
+            dates.find_all{|d| d < Date.today}.map {|date| date.to_s}
+          end
+
+          def transaction_config
+            _config = config.merge(
+              from_date: dates.first.to_s,
+              days: dates.to_a.size,
+              timezone: TIMEZONE,
+              columns: schema
+            )
+            DataSource[*_config.to_a.flatten(1)]
+          end
         end
 
         class TimezoneTest < self
@@ -223,6 +351,13 @@ module Embulk
       end
 
       class RunTest < self
+        def setup_client
+
+          any_instance_of(MixpanelApi::Client) do |klass|
+            stub(klass).export(anything) { records }
+          end
+        end
+
         def setup
           super
 
@@ -286,7 +421,7 @@ module Embulk
           api_secret: API_SECRET,
           timezone: TIMEZONE,
           schema: schema,
-          dates: DATES.to_a,
+          dates: DATES.to_a.map(&:to_s),
           params: Mixpanel.export_params(embulk_config),
         }
       end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: embulk-input-mixpanel
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.2.1
 platform: ruby
 authors:
 - yoshihara
@@ -9,7 +9,7 @@ authors:
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2015-08-17 00:00:00.000000000 Z
+date: 2015-08-26 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement