embulk-input-mixpanel 0.5.14 → 0.5.15

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: cf14d1cd9b35cd1dbde2a915a31fa71b7c321fc3
-  data.tar.gz: 3bd9b09fd763e0b42537f88879146606dbd92d1d
+  metadata.gz: 6c2a00332d19da50d63a726cac679e745206ccab
+  data.tar.gz: f2810c25272ee6b001f206dd0ddd93895c247ab6
 SHA512:
-  metadata.gz: 5c1b12301829f2d451013c0883a06d442fad5321c1b497d7d8761ffd6bd7313ada5fa2c2ede54752a56b2e17f0f324741a449831deb4112625d0200ba2448fd6
-  data.tar.gz: d7fd325a8a6f70c0ff61330f5f3f0f1733242f20f2c5349bd8491ebcfab9c78bbf1a6eb0e870027b5b9ffe61bb15f7fdc073dc979b30278853a9c3ade7894be3
+  metadata.gz: 062a5d9d8948ba9efe802b477756484f03add6ce542d9bdbf40ee3988c2c943f5f6445fc32b15d05dfd0b909761fbf4f47f530e0e2d1357b1a57ca0c20074c55
+  data.tar.gz: df58ccf3ab6acad568681d57c39e97258849dc6e3946f5367ccdf31d64265d8c6d6afbb724259b66d21003b670b1763b6e097a0e871ff9bdaeb9e396f5e1b0cc

data/.travis.yml

@@ -1,6 +1,9 @@
 language: ruby
 
-jdk: oraclejdk8
+dist: trusty
+
+jdk: openjdk8
+
 sudo: false
 
 addons:
@@ -15,9 +18,9 @@ before_install:
     ruby -v
     rvm get head
     rvm use jruby-9.0.5.0 --install
-    gem install bundler
+    gem install bundler -v '< 2'
     ruby -v
-  - gem i bundler
+  - gem install bundler -v '< 2'
 
 rvm:
   - jruby-9.0.5.0

data/CHANGELOG.md

@@ -1,6 +1,10 @@
+## 0.5.15 - 2020-01-22
+
+* [enhancement] Update the authentication method to latest [#63](https://github.com/treasure-data/embulk-input-mixpanel/pull/63)
+
 ## 0.5.14 - 2018-10-22
 
-* [enhancement] Handle the wrong period during transition from standard to daylight saving time exception [#61](https://github.com/treasure-data/embulk-input-mixpanel/pull/61)
+* [enhancement] Handle the wrong period during transition from standard to daylight saving time exception [#62](https://github.com/treasure-data/embulk-input-mixpanel/pull/62)
 
 ## 0.5.13 - 2018-10-04
 

data/README.md

@@ -31,7 +31,6 @@ To get it, you should log in mixpanel website, and click gear icon at the lower
 
 ### Configuration
 
-- **api_key**: project API Key (string, required)
 - **api_secret**: project API Secret (string, required)
 - **export_endpoint**: the Data Export API's endpoint (string, default to "http://data.mixpanel.com/api/2.0/export")
 - **timezone**: project timezone(string, required)
@@ -69,7 +68,6 @@ If you have such data and set config.yml as below.
 ```yaml
 in:
   type: mixpanel
-  api_key: "API_KEY"
   api_secret: "API_SECRET"
   timezone: "US/Pacific"
   from_date: "2015-07-19"
@@ -102,7 +100,6 @@ in:
 ```yaml
 in:
   type: mixpanel
-  api_key: "API_KEY"
   api_secret: "API_SECRET"
   timezone: "US/Pacific"
   from_date: "2015-07-19"

data/embulk-input-mixpanel.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |spec|
   spec.name          = "embulk-input-mixpanel"
-  spec.version       = "0.5.14"
+  spec.version       = "0.5.15"
   spec.authors       = ["yoshihara", "uu59"]
   spec.summary       = "Mixpanel input plugin for Embulk"
   spec.description   = "Loads records from Mixpanel."
@@ -13,7 +13,7 @@ Gem::Specification.new do |spec|
   spec.require_paths = ["lib"]
 
   spec.add_dependency 'httpclient', '>= 2.8.3' # To use tcp_keepalive
-  spec.add_dependency 'tzinfo'
+  spec.add_dependency 'tzinfo', '1.2.5' # to avoid failure in when upgrade to 1.2.6 https://github.com/tzinfo/tzinfo/issues/114
   spec.add_dependency 'perfect_retry', ["~> 0.5"]
   spec.add_dependency 'activesupport'
   spec.add_development_dependency 'bundler', ['~> 1.0']

data/lib/embulk/input/mixpanel.rb

@@ -61,7 +61,6 @@ module Embulk
           dates: range,
           timezone: timezone,
           export_endpoint: export_endpoint(config),
-          api_key: config.param(:api_key, :string),
           api_secret: config.param(:api_secret, :string),
           schema: config.param(:columns, :array),
           fetch_unknown_columns: fetch_unknown_columns,
@@ -129,8 +128,7 @@ module Embulk
           retry_initial_wait_sec: config.param(:retry_initial_wait_sec, :integer, default: 1),
           retry_limit: config.param(:retry_limit, :integer, default: 5),
         })
-        client = MixpanelApi::Client.new(config.param(:api_key, :string),
-                                         config.param(:api_secret, :string),
+        client = MixpanelApi::Client.new(config.param(:api_secret, :string),
                                          retryer,
                                          export_endpoint(config))
 
@@ -162,7 +160,6 @@ module Embulk
 
       def init
         @export_endpoint = task[:export_endpoint]
-        @api_key = task[:api_key]
         @api_secret = task[:api_secret]
         @params = task[:params]
         @timezone = task[:timezone]
@@ -310,7 +307,7 @@ module Embulk
           )
         end
         Embulk.logger.info "Where params is #{params["where"]}"
-        client = MixpanelApi::Client.new(@api_key, @api_secret, self.class.perfect_retry(task), @export_endpoint)
+        client = MixpanelApi::Client.new(@api_secret, self.class.perfect_retry(task), @export_endpoint)
 
         if preview?
           client.export_for_small_dataset(params)
@@ -342,9 +339,7 @@ module Embulk
       def self.export_params(config)
         event = config.param(:event, :array, default: nil)
         event = event.nil? ? nil : event.to_json
-
         {
-          api_key: config.param(:api_key, :string),
           event: event,
           where: config.param(:where, :string, default: nil),
           bucket: config.param(:bucket, :string, default: nil),

data/lib/embulk/input/mixpanel_api/client.rb

@@ -29,7 +29,9 @@ module Embulk
           begin
             retryer.with_retry do
               client = HTTPClient.new
+              client.force_basic_auth = true
               client.connect_timeout = PING_TIMEOUT_SECONDS
+              client.set_auth(nil, @api_secret, nil)
               client.get(URI.join(endpoint, '/'))
             end
             true
@@ -38,9 +40,8 @@ module Embulk
           end
         end
 
-        def initialize(api_key, api_secret, retryer = nil, endpoint = DEFAULT_EXPORT_ENDPOINT)
+        def initialize(api_secret, retryer = nil, endpoint = DEFAULT_EXPORT_ENDPOINT)
           @endpoint = endpoint
-          @api_key = api_key
           @api_secret = api_secret
           @retryer = retryer || PerfectRetry.new do |config|
             # for test
@@ -100,7 +101,6 @@ module Embulk
         def request(params, &block)
           # https://mixpanel.com/docs/api-documentation/exporting-raw-data-you-inserted-into-mixpanel
           Embulk.logger.debug "Export param: #{params.to_s}"
-          set_signatures(params)
 
           buf = ""
           error_response = ''
@@ -133,7 +133,6 @@ module Embulk
           # guess/preview
           # Try to fetch first number of records
           params["limit"] = num_of_records
-          set_signatures(params)
           Embulk.logger.info "Sending request to #{@endpoint}"
           res = httpclient.get(@endpoint, params)
           handle_error(res,res.body)
@@ -154,25 +153,6 @@ module Embulk
           end
         end
 
-        def set_signatures(params)
-          params[:expire] ||= Time.now.to_i + TIMEOUT_SECONDS
-          params[:sig] = signature(params)
-          params
-        end
-
-        def signature(params)
-          # https://mixpanel.com/docs/api-documentation/data-export-api#auth-implementation
-          params.delete(:sig)
-          sorted_keys = params.keys.map(&:to_s).sort
-          signature = sorted_keys.inject("") do |sig, key|
-            value = params[key] || params[key.to_sym]
-            next sig unless value
-            sig << "#{key}=#{value}"
-          end
-
-          Digest::MD5.hexdigest(signature + @api_secret)
-        end
-
         def httpclient
           @client ||=
             begin
@@ -181,6 +161,8 @@ module Embulk
               client.tcp_keepalive = true
               client.default_header = {Accept: "application/json; charset=UTF-8"}
               # client.debug_dev = STDERR
+              client.force_basic_auth = true
+              client.set_auth(nil, @api_secret, nil);
               client
             end
         end

data/test/embulk/input/mixpanel_api/test_client.rb

@@ -8,49 +8,28 @@ module Embulk
       class ClientTest < Test::Unit::TestCase
         include OverrideAssertRaise
 
-        API_KEY = "api_key".freeze
         API_SECRET = "api_secret".freeze
 
         def setup
-          @client = Client.new(API_KEY, API_SECRET)
+          @client = Client.new(API_SECRET)
           stub(Embulk).logger { ::Logger.new(IO::NULL) }
         end
 
-        # NOTE: Client#signature is private method but this value
-        # can't be checked via other methods.
-        def test_signature
-          now = Time.parse("2015-07-22 00:00:00")
-          stub(Time).now { now }
-
-          params = {
-            string: "string",
-            array: ["elem1", "elem2"],
-          }
-          expected = "4be4a4f92f57e12b543a2a5f2f5897b6"
-
-          assert_equal(expected, @client.__send__(:signature, params))
-        end
-
         class TestKeepAlive < self
           def test_tcp_keepalive_enabled
-            client = Client.new(API_KEY, API_SECRET)
+            client = Client.new(API_SECRET)
             assert client.send(:httpclient).tcp_keepalive
           end
         end
 
         class TryToDatesTest < self
           def setup
-            @client = Client.new(API_KEY, API_SECRET)
+            @client = Client.new(API_SECRET)
           end
 
-          data do
-            [
-              ["2000-01-01", "2000-01-01"],
-              ["2010-01-01", "2010-01-01"],
-              ["3 days ago", (Date.today - 3).to_s],
-            ]
-          end
-          def test_candidates(from_str)
+
+          def test_candidates_case1
+            from_str = "2000-01-01"
             from = Date.parse(from_str)
             yesterday = Date.today - 1
             dates = @client.try_to_dates(from.to_s)
@@ -66,6 +45,24 @@ module Embulk
             assert_equal expect, dates
             assert dates.all?{|d| d <= yesterday}
           end
+
+          def test_candidates_case2
+            from_str = (Date.today - 3).to_s
+            from = Date.parse(from_str)
+            yesterday = Date.today - 1
+            dates = @client.try_to_dates(from.to_s)
+            expect = [
+                from + 1,
+                from + 10,
+                from + 100,
+                from + 1000,
+                from + 10000,
+                yesterday,
+            ].find_all{|d| d <= yesterday}
+
+            assert_equal expect, dates
+            assert dates.all?{|d| d <= yesterday}
+          end
         end
 
         class ExportTest < self
@@ -77,7 +74,7 @@ module Embulk
 
           def test_success
             stub_client
-            stub(@client).set_signatures(anything) {}
+            # stub(@client).set_signatures(anything) {}
             stub_response(success_response)
 
             records = []
@@ -90,7 +87,7 @@ module Embulk
 
           def test_export_partial_with_export_terminated_early
             stub_client
-            stub(@client).set_signatures(anything) {}
+            # stub(@client).set_signatures(anything) {}
             stub_response(Struct.new(:code, :body).new(200, jsonl_dummy_responses+"\nexport terminated early"))
 
             records = []
@@ -104,7 +101,7 @@ module Embulk
 
           def test_export_partial_with_error_json
             stub_client
-            stub(@client).set_signatures(anything) {}
+            # stub(@client).set_signatures(anything) {}
             stub_response(Struct.new(:code, :body).new(200, jsonl_dummy_responses+"\n{\"error\":"))
             records = []
             assert_raise MixpanelApi::IncompleteExportResponseError do
@@ -203,7 +200,6 @@ module Embulk
 
           def params
             {
-              "api_key" => API_KEY,
               "api_secret" => API_SECRET,
               "from_date" => "2015-01-01",
               "to_date" => "2015-03-02",

data/test/embulk/input/test_mixpanel.rb

@@ -9,7 +9,6 @@ module Embulk
     class MixpanelTest < Test::Unit::TestCase
       include OverrideAssertRaise
 
-      API_KEY = "api_key".freeze
       API_SECRET = "api_secret".freeze
       FROM_DATE = "2015-02-22".freeze
       TO_DATE = "2015-03-02".freeze
@@ -31,7 +30,6 @@ module Embulk
 
       def setup_client
         params = {
-          api_key: API_KEY,
           event: nil,
           where: nil,
           bucket: nil,
@@ -72,7 +70,6 @@ module Embulk
         def test_from_date_old_date
           config = {
             type: "mixpanel",
-            api_key: API_KEY,
             api_secret: API_SECRET,
             from_date: FROM_DATE,
           }
@@ -87,7 +84,6 @@ module Embulk
         def test_from_date_future
           config = {
             type: "mixpanel",
-            api_key: API_KEY,
             api_secret: API_SECRET,
             timezone: TIMEZONE,
             from_date: (today + 1).to_s
@@ -103,7 +99,6 @@ module Embulk
           from_date = (today - 1).to_s
           config = {
             type: "mixpanel",
-            api_key: API_KEY,
             api_secret: API_SECRET,
             from_date: from_date,
           }
@@ -117,7 +112,6 @@ module Embulk
         def test_no_from_date
           config = {
             type: "mixpanel",
-            api_key: API_KEY,
             api_secret: API_SECRET,
             timezone: TIMEZONE
           }
@@ -141,7 +135,6 @@ module Embulk
           stub(Embulk::Input::MixpanelApi::Client).mixpanel_available? { false }
           config = {
             type: "mixpanel",
-            api_key: API_KEY,
             api_secret: API_SECRET,
           }
 
@@ -300,7 +293,6 @@ module Embulk
           def transaction_task(timezone)
             task.merge(
               dates: DATES.map {|date| date.to_s},
-              api_key: API_KEY,
               api_secret: API_SECRET,
               incremental: true,
               incremental_column: nil,
@@ -366,7 +358,6 @@ module Embulk
             from_date = Date.parse(FROM_DATE)
             task.merge(
               dates: (from_date..(from_date + days - 1)).map {|date| date.to_s},
-              api_key: API_KEY,
               api_secret: API_SECRET,
               timezone: TIMEZONE,
               schema: schema
@@ -434,7 +425,6 @@ module Embulk
         def transaction_task
           task.merge(
             dates: DATES.map {|date| date.to_s},
-            api_key: API_KEY,
             api_secret: API_SECRET,
             timezone: TIMEZONE,
             schema: schema
@@ -451,7 +441,6 @@ module Embulk
       def test_export_params
         config_params = [
           :type, "mixpanel",
-          :api_key, API_KEY,
           :api_secret, API_SECRET,
           :from_date, FROM_DATE,
           :to_date, TO_DATE,
@@ -463,7 +452,6 @@ module Embulk
         config = DataSource[*config_params]
 
         expected = {
-          api_key: API_KEY,
           event: "[\"ViewHoge\",\"ViewFuga\"]",
           where: 'properties["$os"] == "Windows"',
           bucket: "987",
@@ -541,7 +529,6 @@ module Embulk
 
         def task
           {
-            api_key: API_KEY,
             api_secret: API_SECRET,
             export_endpoint: "https://data.mixpanel.com/api/2.0/export/",
             timezone: TIMEZONE,
@@ -846,7 +833,6 @@ module Embulk
 
       def task
         {
-          api_key: API_KEY,
           api_secret: API_SECRET,
           export_endpoint: "https://data.mixpanel.com/api/2.0/export/",
           timezone: TIMEZONE,
@@ -891,7 +877,6 @@ module Embulk
       def config
         {
           type: "mixpanel",
-          api_key: API_KEY,
           api_secret: API_SECRET,
           from_date: FROM_DATE,
           fetch_days: DAYS,

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: embulk-input-mixpanel
 version: !ruby/object:Gem::Version
-  version: 0.5.14
+  version: 0.5.15
 platform: ruby
 authors:
 - yoshihara
@@ -9,7 +9,7 @@ authors:
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2018-10-22 00:00:00.000000000 Z
+date: 2020-02-10 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -28,17 +28,17 @@ dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - '='
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 1.2.5
   name: tzinfo
   prerelease: false
   type: :runtime
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - '='
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 1.2.5
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements: