embulk-input-google_analytics 0.1.17 → 0.1.18.alpha

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 8c0c86f1ec128421575f09495f1650bc36b9003e
-  data.tar.gz: 356d14ca378502fa5a3c7d8781162f318ac62797
+  metadata.gz: 840c722b01ba3cf337f7b5e405e8b6b07676810b
+  data.tar.gz: 9416098303f34773cf3df1329a8eaf84fe4594b2
 SHA512:
-  metadata.gz: 2030bd41d01b4d67e5a694512af6d5cfd4e153570f705e9584fc9d31cb5d7bb439d82726143e6929ffcdc3fb15a1acd2d302d9411a9f3434bf347019b05fc8c7
-  data.tar.gz: d98bf69f0cefa62bb1058104300bba29a1ff527277d562921529f9ce7108bffc4b25fc1a1b8a6920d5f7edbd0f1c1571793c970f862105c91e14c391a0123a01
+  metadata.gz: 736e069eb795ba331c2125df44cb5cc48525f8bb4a44f43e62bb1b0b27c2b1affbd9e8e76fd76dfeeaeb8eaa64b5b9e91cde98254d4a3c90600fb30c7534ac64
+  data.tar.gz: 61a3515ba0cd0c28832d39d215b5619e3e62799164155c28ace126f773c8c4c08398e040b5424d38fb775fd08414d0af20833e8c6d5d4ab038ee5cf1373d459e

data/embulk-input-google_analytics.gemspec

@@ -1,7 +1,7 @@
 
 Gem::Specification.new do |spec|
   spec.name          = "embulk-input-google_analytics"
-	spec.version       = "0.1.17"
+	spec.version       = "0.1.18.alpha"
   spec.authors       = ["uu59"]
   spec.summary       = "Google Analytics input plugin for Embulk"
   spec.description   = "Loads records from Google Analytics."
@@ -27,5 +27,6 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency 'simplecov'
   spec.add_development_dependency "codeclimate-test-reporter"
   spec.add_development_dependency "pry"
+  spec.add_development_dependency "pry-debugger-jruby"
   spec.add_development_dependency "gem_release_helper", "~> 1.0"
 end

data/lib/embulk/input/google_analytics/client.rb

@@ -208,10 +208,22 @@ module Embulk
 
         def auth
           retryer.with_retry do
-            Google::Auth::ServiceAccountCredentials.make_creds(
-              json_key_io: StringIO.new(task["json_key_content"]),
-              scope: "https://www.googleapis.com/auth/analytics.readonly"
-            )
+            case task['auth_method']
+            when Plugin::AUTH_TYPE_JSON_KEY
+              Google::Auth::ServiceAccountCredentials.make_creds(
+                json_key_io: StringIO.new(task["json_key_content"]),
+                scope: "https://www.googleapis.com/auth/analytics.readonly"
+              )
+            when Plugin::AUTH_TYPE_REFRESH_TOKEN
+              Google::Auth::UserRefreshCredentials.new(
+                'token_credential_uri': Google::Auth::UserRefreshCredentials::TOKEN_CRED_URI,
+                'client_id': task['client_id'],
+                'client_secret': task['client_secret'],
+                'refresh_token': task['refresh_token']
+              )
+            else
+              raise Embulk::ConfigError.new("Unknown Authentication method: '#{task['auth_method']}'.")
+            end
           end
         rescue Google::Apis::AuthorizationError => e
           raise ConfigError.new(e.message)

data/lib/embulk/input/google_analytics/plugin.rb

@@ -3,6 +3,8 @@ module Embulk
     module GoogleAnalytics
       class Plugin < InputPlugin
         ::Embulk::Plugin.register_input("google_analytics", self)
+        AUTH_TYPE_JSON_KEY = 'json_key'.freeze
+        AUTH_TYPE_REFRESH_TOKEN = 'refresh_token'.freeze
 
         # https://developers.google.com/analytics/devguides/reporting/core/dimsmets
 
@@ -11,6 +13,15 @@ module Embulk
           unless %w(ga:date ga:dateHour).include?(task["time_series"])
             raise ConfigError.new("Unknown time_series '#{task["time_series"]}'. Use 'ga:dateHour' or 'ga:date'")
           end
+
+          raise ConfigError.new("Unknown Authentication method '#{task['auth_method']}'.") unless task['auth_method']
+
+          if task['auth_method'] == Plugin::AUTH_TYPE_REFRESH_TOKEN
+            unless task['client_id'] && task['client_secret'] && task['refresh_token']
+              raise ConfigError.new("client_id, client_secret and refresh_token are required when using Oauth authentication")
+            end
+          end
+
           columns_list = Client.new(task).get_columns_list
 
           columns = columns_from_task(task).map do |col_name|
@@ -53,8 +64,16 @@ module Embulk
         end
 
         def self.task_from_config(config)
-          json_key_content = config.param("json_key_content", :string)
-          {
+          refresh_token = config.param('refresh_token', :string, default: nil)
+          json_key_content = config.param("json_key_content", :string, default: nil)
+
+          auth_method = Plugin::AUTH_TYPE_REFRESH_TOKEN if refresh_token
+          auth_method = Plugin::AUTH_TYPE_JSON_KEY if json_key_content && auth_method == nil
+            {
+            "auth_method" => auth_method,
+            "client_id" => config.param("client_id", :string, default: nil),
+            "client_secret" => config.param("client_secret", :string, default: nil),
+            "refresh_token" => refresh_token,
             "json_key_content" => json_key_content,
             "view_id" => config.param("view_id", :string),
             "dimensions" => config.param("dimensions", :array, default: []),
@@ -98,7 +117,7 @@ module Embulk
           client = Client.new(task, preview?)
           columns = self.class.columns_from_task(task) + ["view_id"]
 
-          last_record_time = task["last_record_time"] ? Time.parse(task["last_record_time"]) : nil
+          last_record_time = task['incremental'] && task["last_record_time"] ? Time.parse(task["last_record_time"]) : nil
 
           latest_time_series = nil
           skip_counter, total_counter = 0, 0

data/test/embulk/input/google_analytics/test_client.rb

@@ -283,6 +283,60 @@ module Embulk
           end
         end
 
+        sub_test_case "oauth_refresh_token" do
+          setup do
+            conf = valid_config_oauth["in"]
+            mute_logger
+            @client = Client.new(task(embulk_config(conf)))
+          end
+
+          sub_test_case "retry refresh token" do
+            def should_retry
+              mock(Google::Auth::UserRefreshCredentials).new(anything).times(retryer.config.limit + 1) { raise error }
+              assert_raise do
+                @client.auth
+              end
+            end
+
+            def should_not_retry
+              mock(Google::Auth::UserRefreshCredentials).new(anything).times(1) { raise error }
+              assert_raise do
+                @client.auth
+              end
+            end
+
+            sub_test_case "Server respones error (5xx)" do
+              def error
+                Google::Apis::ServerError.new("error")
+              end
+
+              test "should retry refresh token" do
+                should_retry
+              end
+            end
+
+            sub_test_case "Refresh token reach Rate Limit" do
+              def error
+                Google::Apis::RateLimitError.new("error")
+              end
+
+              test "should retry refresh token" do
+                should_retry
+              end
+            end
+
+            sub_test_case "Refresh token Error" do
+              def error
+                Google::Apis::AuthorizationError.new("error")
+              end
+
+              test "should not retry" do
+                should_not_retry
+              end
+            end
+          end
+        end
+
         sub_test_case "too_early_data?" do
           def stub_timezone(client)
             stub(client).get_profile { {timezone: "America/Los_Angeles" } }
@@ -412,6 +466,10 @@ module Embulk
           fixture_load("valid.yml")
         end
 
+        def valid_config_oauth
+          fixture_load("valid_refresh_token.yml")
+        end
+
         def embulk_config(hash)
           Embulk::DataSource.new(hash)
         end

data/test/embulk/input/google_analytics/test_plugin.rb

@@ -75,6 +75,46 @@ module Embulk
               end
             end
 
+            test "missing client_id params for oauth" do
+              conf = valid_config_oauth["in"]
+              conf.delete("client_id")
+              assert_raise(Embulk::ConfigError.new(oauth_expect_message)) do
+                Plugin.transaction(embulk_config(conf))
+              end
+            end
+
+            test "missing client_secret params for oauth" do
+              conf = valid_config_oauth["in"]
+              conf.delete('client_secret')
+              assert_raise(Embulk::ConfigError.new(oauth_expect_message)) do
+                Plugin.transaction(embulk_config(conf))
+              end
+            end
+
+            test "missing refresh_token params for oauth" do
+              conf = valid_config_oauth["in"]
+              conf.delete("refresh_token")
+              assert_raise(Embulk::ConfigError.new(unknown_auth_method_message)) do
+                Plugin.transaction(embulk_config(conf))
+              end
+            end
+
+            test "missing json_key_content params for oauth" do
+              conf = valid_config["in"]
+              conf.delete("json_key_content")
+              assert_raise(Embulk::ConfigError.new(unknown_auth_method_message)) do
+                Plugin.transaction(embulk_config(conf))
+              end
+            end
+
+            def unknown_auth_method_message
+              "Unknown Authentication method ''."
+            end
+
+            def oauth_expect_message
+              "client_id, client_secret and refresh_token are required when using Oauth authentication"
+            end
+
             def unknown_col_name
               "ga:foooooo"
             end
@@ -162,12 +202,14 @@ module Embulk
 
               sub_test_case "last_record_time option" do
                 setup do
+                  stub(Plugin).calculate_next_times { {} }
                   Time.zone = "America/Los_Angeles"
                   @last_record_time = Time.zone.parse("2016-06-01 12:00:00").to_time
 
                   conf = valid_config["in"]
                   conf["time_series"] = time_series
                   conf["last_record_time"] = @last_record_time.strftime("%Y-%m-%d %H:%M:%S %z")
+                  conf['incremental'] = true
                   @plugin = Plugin.new(embulk_config(conf), nil, nil, @page_builder)
                 end
 
@@ -182,6 +224,7 @@ module Embulk
                       }
                       block.call row
                     end
+                    stub(klass).get_profile { {timezone: "America/Los_Angeles" } }
                   end
 
                   mock(@page_builder).add.never
@@ -219,12 +262,15 @@ module Embulk
 
               sub_test_case "last_record_time option" do
                 setup do
+                  stub(Plugin).calculate_next_times { {} }
                   Time.zone = "America/Los_Angeles"
                   @last_record_time = Time.zone.parse("2016-06-01 12:00:00").to_time
 
                   conf = valid_config["in"]
                   conf["time_series"] = time_series
                   conf["last_record_time"] = @last_record_time.strftime("%Y-%m-%d %H:%M:%S %z")
+                  conf['incremental'] = true
+
                   @plugin = Plugin.new(embulk_config(conf), nil, nil, @page_builder)
                 end
 
@@ -239,6 +285,7 @@ module Embulk
                       }
                       block.call row
                     end
+                    stub(klass).get_profile { {timezone: "America/Los_Angeles" } }
                   end
 
                   mock(@page_builder).add.never
@@ -250,6 +297,39 @@ module Embulk
           end
         end
 
+        sub_test_case "run non-incremental" do
+          sub_test_case " should be added to page_builder" do
+            setup do
+              Time.zone = "America/Los_Angeles"
+              @last_record_time = Time.zone.parse("2016-06-01 12:00:00").to_time
+
+              conf = valid_config["in"]
+              conf["time_series"] = "ga:date"
+              conf["last_record_time"] = @last_record_time.strftime("%Y-%m-%d %H:%M:%S %z")
+              conf['incremental'] = false
+              @plugin = Plugin.new(embulk_config(conf), nil, nil, @page_builder)
+            end
+
+            test "should allow record to be added to page_builder" do
+              any_instance_of(Client) do |klass|
+                stub(klass).each_report_row do |block|
+                  row = {
+                      "ga:date" => @last_record_time,
+                      "ga:browser" => "wget",
+                      "ga:visits" => 3,
+                      "ga:pageviews" => 4,
+                  }
+                  block.call row
+                end
+              end
+
+              mock(@page_builder).add(anything).at_least(1)
+              mock(@page_builder).finish
+              @plugin.run
+            end
+          end
+        end
+
         sub_test_case "canonicalize_column_name" do
           data do
             [
@@ -457,6 +537,10 @@ module Embulk
         def embulk_config(hash)
           Embulk::DataSource.new(hash)
         end
+
+        def valid_config_oauth
+          fixture_load("valid_refresh_token.yml")
+        end
       end
     end
   end

data/test/fixtures/valid_refresh_token.yml

@@ -0,0 +1,18 @@
+in:
+  type: google_analytics
+  refresh_token: "your_refresh_token"
+  client_id: "your_client_id_here"
+  client_secret: "your_client_secret_here"
+  view_id: 101111111
+  time_series: "ga:dateHour"
+ 
+  dimensions:
+    - "ga:browser"
+  metrics:
+    - "ga:visits"
+    - "ga:pageviews"
+  retry_limit: 2
+  retry_initial_wait_sec: 0
+
+out:
+  type: stdout

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk-input-google_analytics
 version: !ruby/object:Gem::Version
-  version: 0.1.17
+  version: 0.1.18.alpha
 platform: ruby
 authors:
 - uu59
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2018-04-16 00:00:00.000000000 Z
+date: 2018-05-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -192,6 +192,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  name: pry-debugger-jruby
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
@@ -231,6 +245,7 @@ files:
 - test/fixture_helper.rb
 - test/fixtures/reports.json
 - test/fixtures/valid.yml
+- test/fixtures/valid_refresh_token.yml
 - test/override_assert_raise.rb
 - test/run-test.rb
 homepage: https://github.com/treasure-data/embulk-input-google_analytics
@@ -248,9 +263,9 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">="
+  - - ">"
     - !ruby/object:Gem::Version
-      version: '0'
+      version: 1.3.1
 requirements: []
 rubyforge_project:
 rubygems_version: 2.6.6
@@ -263,5 +278,6 @@ test_files:
 - test/fixture_helper.rb
 - test/fixtures/reports.json
 - test/fixtures/valid.yml
+- test/fixtures/valid_refresh_token.yml
 - test/override_assert_raise.rb
 - test/run-test.rb