embulk-input-mixpanel 0.1.0 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: d553d9b694f3b3150c2c634f1f4b334e7c2162e9
-  data.tar.gz: 89480ccc7dc13ace59cb88769af0263b0c078c72
+  metadata.gz: 5fca90578e059bd058fa52d1d8283afbd85a58a1
+  data.tar.gz: 09305de32323e4284f33a005330d522f2afc98e7
 SHA512:
-  metadata.gz: 60063447a33ad85b84d33bb01d85ad757c3bcc0494908f30a9c8c78e8fb5df17049c27833b1ce6acccf52358b6575b036ebdbd770fbff4e38558ecdd90734120
-  data.tar.gz: d98177f66557bec6a845749bf93ac69a4b4b8bbbbf63695222f945d37b1d6832432eb58c8448b354018c819e676d5fcf48409b723a32fd488291fd0b724f6bd6
+  metadata.gz: 8a200ccc100c2baa7552e933846648253bace0b944a7cefa8bfcc87198ab0e5dfe2a8a754a3195ac2aeca096114a92defc74c3d7e35176a2d8475bcac830cf83
+  data.tar.gz: 91c0591f7bf83a14dc5bdb7f56f2915c97ba8e3d3b81cac8f37ac411541c0ae5302bf2712c6f4e6347a542955c5278a085cab67925a8da6559ce0195dc12edaf

data/.travis.yml

@@ -13,4 +13,13 @@ jdk:
 gemfile:
   - gemfiles/embulk-0.6.16
   - gemfiles/embulk-0.6.17
+  - gemfiles/embulk-0.6.18
+  - gemfiles/embulk-0.6.19
+  - gemfiles/embulk-0.6.20
+  - gemfiles/embulk-0.6.21
+  - gemfiles/embulk-0.6.22
+  - gemfiles/embulk-0.6.23
   - gemfiles/embulk-latest
+matrix:
+  allow_failures:
+    - gemfile: gemfiles/embulk-0.6.22

data/CHANGELOG.md

@@ -1,3 +1,11 @@
+## 0.2.0 - 2015-08-17
+
+* [enhanement] Raise config error for unretryable [#15](https://github.com/treasure-data/embulk-input-mixpanel/pull/15) [[Reported by @muga](https://github.com/treasure-data/embulk-input-mixpanel/issues/11). Thanks!!]
+* [maintenance] Use everyleaf-embulk_helper [#14](https://github.com/treasure-data/embulk-input-mixpanel/pull/14)
+* [enhancement] Support scheduled execution [#13](https://github.com/treasure-data/embulk-input-mixpanel/pull/13) [[Reported by @muga](https://github.com/treasure-data/embulk-input-mixpanel/issues/12). Thanks!!]
+* [maintenance] Improve coverage [#10](https://github.com/treasure-data/embulk-input-mixpanel/pull/10)
+* [fixed] README: Add description for how to get project's timezone [#9](https://github.com/treasure-data/embulk-input-mixpanel/pull/9)
+
 ## 0.1.0 - 2015-07-28
 
 The first release!!

data/README.md

@@ -23,13 +23,19 @@ This plugin uses API key and API secret for target project. Before you make your
 
 For API configuration, you should log in mixpanel website, and click "Account" at the header. When you select "Projects" panel, you can get "API Key" and "API Secret" for each project.
 
+### How to get project's timezone
+
+This plugin uses project's timezone to adjust timestamp to UTC.
+
+To get it, you should log in mixpanel website, and click gear icon at the lower left. Then an opened dialog shows timezone at "Timezone" column in "Management" tab.
+
 ### Configuration
 
 - **api_key**: project API Key (string, required)
 - **api_secret**: project API Secret (string, required)
 - **timezone**: project timezone(string, required)
-- **from_date**: From date to export (string, required)
-- **to_date**: To date to export (string, required)
+- **from_date**: From date to export (string, optional, default: today - 2)
+- **days**: Count of days range for exporting (integer, optional, default: (today - 1) - from_date)
 - **event**: The event or events to filter data (array, optional, default: nil)
 - **where**: Expression to filter data (c.f. https://mixpanel.com/docs/api-documentation/data-export-api#segmentation-expressions) (string, optional, default: nil)
 - **bucket**:The data backet to filter data (string, optional, default: nil)
@@ -43,7 +49,7 @@ in:
   api_secret: "API_SECRET"
   timezone: "US/Pacific"
   from_date: "2015-07-19"
-  to_date: "2015-07-20"
+  days: 5
 ```
 
 ## Run test

data/Rakefile

@@ -1,4 +1,5 @@
 require "bundler/gem_tasks"
+require "everyleaf/embulk_helper/tasks"
 
 task default: :test
 
@@ -7,54 +8,7 @@ task :test do
   ruby("test/run-test.rb", "--use-color=yes", "--collector=dir")
 end
 
-namespace :release do
-  desc "Add header of now version release to ChangeLog and bump up version"
-  task :prepare do
-    root_dir = Pathname.new(File.expand_path("../", __FILE__))
-    changelog_file = root_dir.join("CHANGELOG.md")
-    gemspec_file = root_dir.join("embulk-input-mixpanel.gemspec")
-
-    system("git fetch origin")
-
-    # detect merged PR
-    old_version = gemspec_file.read[/spec\.version += *"([0-9]+\.[0-9]+\.[0-9]+)"/, 1]
-    pr_numbers = `git log v#{old_version}..origin/master --oneline`.scan(/#[0-9]+/)
-
-    if !$?.success? || pr_numbers.empty?
-      puts "Detecting PR failed. Please confirm if any PR were merged after the latest release."
-      exit(false)
-    end
-
-    # Generate new version
-    major, minor, patch = old_version.split(".").map(&:to_i)
-    new_version = "#{major}.#{minor}.#{patch + 1}"
-
-    # Update ChangeLog
-    pr_descriptions = pr_numbers.map do |number|
-      body = open("https://api.github.com/repos/treasure-data/embulk-input-mixpanel/issues/#{number.gsub("#", "")}").read
-      payload = JSON.parse(body)
-      "* [] #{payload["title"]} [#{number}](https://github.com/treasure-data/embulk-input-mixpanel/pull/#{number.gsub('#', '')})"
-    end.join("\n")
-
-    new_changelog = <<-HEADER
-## #{new_version} - #{Time.now.strftime("%Y-%m-%d")}
-#{pr_descriptions}
-
-#{changelog_file.read.chomp}
-HEADER
-
-    File.open(changelog_file, "w") {|f| f.write(new_changelog) }
-
-    # Update version.rb
-    old_content = gemspec_file.read
-    File.open(gemspec_file, "w") do |f|
-      f.write old_content.gsub(/(spec\.version += *)".*?"/, %Q!\\1"#{new_version}"!)
-    end
-
-    # Update Gemfile.lock
-    system("bundle install")
-
-    puts "ChangeLog, version and Gemfile.lock were updated. New version is #{new_version}."
-  end
-end
-
+Everyleaf::EmbulkHelper::Tasks.install(
+  gemspec: "./embulk-input-mixpanel.gemspec",
+  github_name: "treasure-data/embulk-input-mixpanel",
+)

data/embulk-input-mixpanel.gemspec

@@ -1,7 +1,7 @@
 
 Gem::Specification.new do |spec|
   spec.name          = "embulk-input-mixpanel"
-  spec.version       = "0.1.0"
+  spec.version       = "0.2.0"
   spec.authors       = ["yoshihara", "uu59"]
   spec.summary       = "Mixpanel input plugin for Embulk"
   spec.description   = "Loads records from Mixpanel."
@@ -22,4 +22,5 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency 'test-unit'
   spec.add_development_dependency 'test-unit-rr'
   spec.add_development_dependency 'codeclimate-test-reporter'
+  spec.add_development_dependency 'everyleaf-embulk_helper'
 end

data/gemfiles/embulk-0.6.18

@@ -0,0 +1,4 @@
+source 'https://rubygems.org/'
+gemspec :path => '../'
+
+gem "embulk", "0.6.18"

data/gemfiles/embulk-0.6.19

@@ -0,0 +1,4 @@
+source 'https://rubygems.org/'
+gemspec :path => '../'
+
+gem "embulk", "0.6.19"

data/gemfiles/embulk-0.6.20

@@ -0,0 +1,4 @@
+source 'https://rubygems.org/'
+gemspec :path => '../'
+
+gem "embulk", "0.6.20"

data/gemfiles/embulk-0.6.21

@@ -0,0 +1,4 @@
+source 'https://rubygems.org/'
+gemspec :path => '../'
+
+gem "embulk", "0.6.21"

data/gemfiles/embulk-0.6.22

@@ -0,0 +1,4 @@
+source 'https://rubygems.org/'
+gemspec :path => '../'
+
+gem "embulk", "0.6.22"

data/gemfiles/embulk-0.6.23

@@ -0,0 +1,4 @@
+source 'https://rubygems.org/'
+gemspec :path => '../'
+
+gem "embulk", "0.6.23"

data/gemfiles/embulk-latest

@@ -1,2 +1,4 @@
 source 'https://rubygems.org/'
 gemspec :path => '../'
+
+gem "embulk", "> 0.6.16"

data/gemfiles/template.erb

@@ -0,0 +1,4 @@
+source 'https://rubygems.org/'
+gemspec :path => '../'
+
+gem "embulk", "<%= version %>"

data/lib/embulk/input/mixpanel.rb

@@ -3,7 +3,6 @@ require "embulk/input/mixpanel_api/client"
 
 module Embulk
   module Input
-
     class Mixpanel < InputPlugin
       Plugin.register_input("mixpanel", self)
 
@@ -19,9 +18,7 @@ module Embulk
 
         task[:params] = export_params(config)
 
-        from_date = config.param(:from_date, :string)
-        to_date = config.param(:to_date, :string)
-        dates = Date.parse(from_date)..Date.parse(to_date)
+        dates = generate_dates(config)
         task[:dates] = dates.map {|date| date.to_s}
 
         task[:api_key] = config.param(:api_key, :string)
@@ -33,7 +30,7 @@ module Embulk
           TZInfo::Timezone.get(task[:timezone])
         rescue => e
           Embulk.logger.error "'#{task[:timezone]}' is invalid timezone"
-          raise e
+          raise ConfigError, e.message
         end
 
         columns = []
@@ -51,7 +48,12 @@ module Embulk
       def self.resume(task, columns, count, &control)
         commit_reports = yield(task, columns, count)
 
-        next_config_diff = {}
+        # NOTE: If this plugin supports to run by multi threads, this
+        # implementation is terrible.
+        commit_report = commit_reports.first
+        next_to_date = Date.parse(commit_report[:to_date]).next
+
+        next_config_diff = {from_date: next_to_date.to_s}
         return next_config_diff
       end
 
@@ -83,6 +85,25 @@ module Embulk
         return {"columns" => columns}
       end
 
+      def self.generate_dates(config)
+        default_from_date = (Date.today - 2).to_s
+
+        begin
+          from_date = Date.parse(config.param(:from_date, :string, default: default_from_date))
+        rescue ArgumentError # invalid date
+          raise ConfigError, "from_date '#{from_date}' is invalid date"
+        end
+
+        default_days = ((Date.today - 1) - from_date).to_i
+        days = config.param(:days, :integer, default: default_days)
+
+        if days < 1
+          raise ConfigError, "days '#{days}' is invalid. Please spcify bigger number than 0."
+        end
+
+        from_date..(from_date + days)
+      end
+
       def init
         @api_key = task[:api_key]
         @api_secret = task[:api_secret]
@@ -126,7 +147,7 @@ module Embulk
 
         page_builder.finish
 
-        commit_report = {}
+        commit_report = {to_date: @dates.last}
         return commit_report
       end
 

data/lib/embulk/input/mixpanel_api/client.rb

@@ -21,9 +21,10 @@ module Embulk
           params[:sig] = signature(params)
           response = httpclient.get(ENDPOINT_EXPORT, params)
 
-          if response.code >= 400
-            Embulk.logger.error response.body
-            return Enumerator.new{ }
+          if (400..499).include?(response.code)
+            raise ConfigError, response.body
+          elsif response.code >= 500
+            raise RuntimeError, response.body
           end
 
           Enumerator.new do |y|

data/test/embulk/input/mixpanel_api/test_client.rb

@@ -31,10 +31,17 @@ module Embulk
             super
 
             @httpclient = HTTPClient.new
-            stub(@client).httpclient { @httpclient }
+          end
+
+          def test_httpclient
+            stub_response(success_response)
+            mock(@client).httpclient { @httpclient }
+
+            @client.export(params)
           end
 
           def test_response_class
+            stub_client
             stub_response(success_response)
 
             actual = @client.export(params)
@@ -43,6 +50,7 @@ module Embulk
           end
 
           def test_http_request
+            stub_client
             mock(@httpclient).get(Client::ENDPOINT_EXPORT, params) do
               success_response
             end
@@ -51,6 +59,7 @@ module Embulk
           end
 
           def test_success
+            stub_client
             stub_response(success_response)
 
             actual = @client.export(params)
@@ -58,24 +67,30 @@ module Embulk
             assert_equal(dummy_responses, actual.to_a)
           end
 
-          def test_failure
-            stub_response(failure_response)
-
-            stub(Embulk.logger).error(failure_response.body) {}
-            actual = @client.export(params)
+          def test_failure_with_400
+            stub_client
+            stub_response(failure_response(400))
 
-            assert_equal([], actual.to_a)
+            assert_raise(Embulk::ConfigError) do
+              @client.export(params)
+            end
           end
 
-          def test_failure_logging
-            stub_response(failure_response)
+          def test_failure_with_500
+            stub_client
+            stub_response(failure_response(500))
 
-            mock(Embulk.logger).error(failure_response.body) {}
-            @client.export(params)
+            assert_raise(RuntimeError) do
+              @client.export(params)
+            end
           end
 
           private
 
+          def stub_client
+            stub(@client).httpclient { @httpclient }
+          end
+
           def stub_response(response)
             stub(@httpclient).get(Client::ENDPOINT_EXPORT, params) do
               response
@@ -86,8 +101,8 @@ module Embulk
             Struct.new(:code, :body).new(200, jsonl_dummy_responses)
           end
 
-          def failure_response
-            Struct.new(:code, :body).new(400, "{'error': 'invalid'}")
+          def failure_response(code)
+            Struct.new(:code, :body).new(code, "{'error': 'invalid'}")
           end
 
           def params

data/test/embulk/input/test_mixpanel.rb

@@ -9,6 +9,9 @@ module Embulk
       API_SECRET = "api_secret".freeze
       FROM_DATE = "2015-02-22".freeze
       TO_DATE = "2015-03-02".freeze
+      DAYS = 8
+      DATES = Date.parse(FROM_DATE)..(Date.parse(FROM_DATE) + DAYS)
+      TIMEZONE = "Asia/Tokyo".freeze
 
       DURATIONS = [
         {from_date: FROM_DATE, to_date: "2015-02-28"}, # It has 7 days between 2015-02-22 and 2015-02-28
@@ -56,6 +59,144 @@ module Embulk
         assert_equal(expected, actual)
       end
 
+      class GenerateDatesTest < self
+        def test_valid_from_date
+          from_date = "2015-08-14"
+          parsed_date = Date.parse(from_date)
+
+          expected = (parsed_date..(parsed_date + DAYS))
+          actual = Mixpanel.generate_dates(transaction_config(from_date))
+          assert_equal(expected, actual)
+        end
+
+        def test_invalid_from_date
+          from_date = "2015-08-41"
+
+          assert_raise(Embulk::ConfigError) do
+            Mixpanel.generate_dates(transaction_config(from_date))
+          end
+        end
+
+        private
+
+        def transaction_config(from_date)
+          _config = config.merge(
+            from_date: from_date,
+            timezone: TIMEZONE,
+            columns: schema,
+          )
+          DataSource[*_config.to_a.flatten(1)]
+        end
+      end
+
+      class TransactionTest < self
+        class FromDateTest < self
+        end
+
+        class TimezoneTest < self
+          def test_valid_timezone
+            timezone = TIMEZONE
+            mock(Mixpanel).resume(transaction_task(timezone), columns, 1, &control)
+
+            Mixpanel.transaction(transaction_config(timezone), &control)
+          end
+
+          def test_invalid_timezone
+            timezone = "#{TIMEZONE}ooooo"
+
+            assert_raise(Embulk::ConfigError) do
+              Mixpanel.transaction(transaction_config(timezone), &control)
+            end
+          end
+
+          private
+
+          def transaction_task(timezone)
+            task.merge(
+              dates: DATES.map {|date| date.to_s},
+              api_key: API_KEY,
+              api_secret: API_SECRET,
+              timezone: timezone,
+              schema: schema
+            )
+          end
+
+          def transaction_config(timezone)
+            _config = config.merge(
+              timezone: timezone,
+              columns: schema,
+            )
+            DataSource[*_config.to_a.flatten(1)]
+          end
+        end
+
+        class DaysTest < self
+          def test_valid_days
+            days = 5
+
+            mock(Mixpanel).resume(transaction_task(days), columns, 1, &control)
+            Mixpanel.transaction(transaction_config(days), &control)
+          end
+
+          def test_invalid_days
+            days = 0
+
+            assert_raise(Embulk::ConfigError) do
+              Mixpanel.transaction(transaction_config(days), &control)
+            end
+          end
+
+          private
+
+          def transaction_task(days)
+            from_date = Date.parse(FROM_DATE)
+            task.merge(
+              dates: (from_date..(from_date + days)).map {|date| date.to_s},
+              api_key: API_KEY,
+              api_secret: API_SECRET,
+              timezone: TIMEZONE,
+              schema: schema
+            )
+          end
+
+          def transaction_config(days)
+            _config = config.merge(
+              days: days,
+              columns: schema,
+              timezone: TIMEZONE,
+            )
+            DataSource[*_config.to_a.flatten(1)]
+          end
+        end
+
+        def test_resume
+          today = Date.today
+          control = proc { [{to_date: today.to_s}] }
+          actual = Mixpanel.resume(transaction_task, columns, 1, &control)
+          assert_equal({from_date: today.next.to_s}, actual)
+        end
+
+        def control
+          proc {} # dummy
+        end
+
+        def transaction_task
+          task.merge(
+            dates: DATES.map {|date| date.to_s},
+            api_key: API_KEY,
+            api_secret: API_SECRET,
+            timezone: TIMEZONE,
+            schema: schema
+          )
+        end
+
+        def columns
+          schema.map do |col|
+            Column.new(nil, col["name"], col["type"].to_sym)
+          end
+        end
+      end
+
       def test_export_params
         config_params = [
           :type, "mixpanel",
@@ -89,6 +230,14 @@ module Embulk
           @plugin = Mixpanel.new(task, nil, nil, @page_builder)
         end
 
+        def test_preview_check
+          mock(@plugin).preview? { true }
+          stub(@page_builder).add(anything)
+          stub(@page_builder).finish
+
+          @plugin.run
+        end
+
         def test_preview
           stub(@plugin).preview? { true }
           mock(@page_builder).add(anything).times(records.length)
@@ -114,28 +263,8 @@ module Embulk
           @plugin.run
         end
 
-        def test_invalid_timezone
-          assert_raise(TZInfo::InvalidTimezoneIdentifier) do
-            Mixpanel.new(task.merge(timezone: "Asia/Tokyooooooooo"), nil, nil, @page_builder).run
-          end
-        end
-
         private
 
-        def task
-          {
-            api_key: API_KEY,
-            api_secret: API_SECRET,
-            timezone: "Asia/Tokyo",
-            schema: [
-              {"name" => "foo", "type" => "long"},
-              {"name" => "time", "type" => "long"},
-            ],
-            dates: (Date.parse(FROM_DATE)..Date.parse(TO_DATE)).to_a,
-            params: Mixpanel.export_params(embulk_config),
-          }
-        end
-
         def timezone_offset_seconds
           60 * 60 * 9 # Asia/Tokyo
         end
@@ -143,6 +272,25 @@ module Embulk
 
       private
 
+      def schema
+        [
+          {"name" => "foo", "type" => "long"},
+          {"name" => "time", "type" => "long"},
+          {"name" => "event", "type" => "string"},
+        ]
+      end
+
+      def task
+        {
+          api_key: API_KEY,
+          api_secret: API_SECRET,
+          timezone: TIMEZONE,
+          schema: schema,
+          dates: DATES.to_a,
+          params: Mixpanel.export_params(embulk_config),
+        }
+      end
+
       def records
         [
           {
@@ -160,14 +308,17 @@ module Embulk
         1234567890
       end
 
-      def embulk_config
-        config = {
+      def config
+        {
           type: "mixpanel",
           api_key: API_KEY,
           api_secret: API_SECRET,
           from_date: FROM_DATE,
-          to_date: TO_DATE,
+          days: DAYS,
         }
+      end
+
+      def embulk_config
         DataSource[*config.to_a.flatten(1)]
       end
     end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: embulk-input-mixpanel
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.2.0
 platform: ruby
 authors:
 - yoshihara
@@ -9,7 +9,7 @@ authors:
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2015-07-28 00:00:00.000000000 Z
+date: 2015-08-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -143,6 +143,20 @@ dependencies:
     - - '>='
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  name: everyleaf-embulk_helper
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 description: Loads records from Mixpanel.
 email:
 - h.yoshihara@everyleaf.com
@@ -161,7 +175,14 @@ files:
 - embulk-input-mixpanel.gemspec
 - gemfiles/embulk-0.6.16
 - gemfiles/embulk-0.6.17
+- gemfiles/embulk-0.6.18
+- gemfiles/embulk-0.6.19
+- gemfiles/embulk-0.6.20
+- gemfiles/embulk-0.6.21
+- gemfiles/embulk-0.6.22
+- gemfiles/embulk-0.6.23
 - gemfiles/embulk-latest
+- gemfiles/template.erb
 - lib/embulk/input/mixpanel.rb
 - lib/embulk/input/mixpanel_api/client.rb
 - test/embulk/input/mixpanel_api/test_client.rb