embulk-input-google_analytics 0.1.18 → 0.1.24

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 1ab44fb8b7d3108fc5611a58e38640eb2839478b
-  data.tar.gz: c7dbaea282c1703273cffaf02799c5fc2a4a9013
+  metadata.gz: 2eea056ceeddbfd441fd63830d0dd80d1f145094
+  data.tar.gz: e40d6249b25c5752c10f107f3ea0259f24b8c859
 SHA512:
-  metadata.gz: 42e3f0dedf09f963e1e8d016cc673f4c5737a41f2a708e6b5f2245769862c71ba2bd50ca79b622c66a7b896135c18f7568e1db4c059d0ba42de359c02dcae4e8
-  data.tar.gz: 4e9e5f619ae9ddf6acfdd413984533f14d28cfbb2a59f34bf351b37a128e665e411a2f3cc49092cbf3b3841de9b099d85b42cd9b112bc7d7cc5413f6f2d32f67
+  metadata.gz: 569b256dc6577da275bf8c4be9806d7310c7fff1cd7dfbf24cced151b79e828335b566b4fcea96603c650cd7b09584d38438081d68e287c36a6744b5e80fb0f6
+  data.tar.gz: 7f235dc9a9ea8749261e36315aac61edc864a89c45ff90974b2184aebe14e6a0b75b20127cd23c8a31b8af4cb11953a6c9d9e5a42054c6f176ac826944f44dec

data/.circleci/config.yml

@@ -0,0 +1,37 @@
+version: 2
+jobs:
+  build:
+    parallelism: 1
+    environment:
+      JRUBY_OPTS: "-Xcli.debug=true --debug"
+    docker:
+      - image: circleci/jruby:9.1.17.0
+    working_directory: ~/workspace
+    steps:
+    - checkout
+    - restore_cache:
+        keys:
+          - deps-{{ .Environment.CIRCLE_JOB }}-{{ checksum "embulk-input-google_analytics.gemspec" }}
+    - run: 
+        name: Install bundle
+        command: |
+          ruby -v
+          gem install bundler
+    - run: 
+        name: Install dependencies
+        command: |
+          bundle -v
+          bundle config https://treasuredata.jfrog.io $TD_ARTIFACTORY_USERNAME:$TD_ARTIFACTORY_PASSWORD
+          bundle install --path=vendor/bundle
+    - save_cache:
+        paths:
+          - vendor/bundle
+        key: deps-{{ .Environment.CIRCLE_JOB }}-{{ checksum "embulk-input-google_analytics.gemspec" }}
+    - run: 
+        name: Build
+        command: |
+          bundle exec rake build
+    - run: 
+        name: Execute tests
+        command: |
+          bundle exec rake

data/.gitignore

@@ -4,3 +4,4 @@
 /.bundle/
 /Gemfile.lock
 /coverage/
+vendor/

data/CHANGELOG.md

@@ -1,3 +1,19 @@
+## 0.1.24 - 2020-07-14
+Upgrade version of gems to be compatible with other google input plugins [#46](https://github.com/treasure-data/embulk-input-google_analytics/pull/46)
+Release version 0.1.24 https://github.com/treasure-data/embulk-input-google_analytics/commit/3aef789a672227ae3ad9b58235091d34c96a4b67
+
+## 0.1.23 - 2019-10-24
+Lock signet and activesupport by less than or equal locked version [#45](https://github.com/treasure-data/embulk-input-google_analytics/pull/45)
+
+## 0.1.22 - 2019-10-24
+* Fix **Plugin's dependencies require Ruby > 2.3** [#44](https://github.com/treasure-data/embulk-input-google_analytics/pull/44)
+
+## 0.1.21 - 2019-04-01
+* Fix **(ArgumentError) no time information in ""** when last_record_time is null [#42](https://github.com/treasure-data/embulk-input-google_analytics/pull/42)
+
+## 0.1.19 - 2018-10-03
+* Make it easy to understand error message of Google Analytics connector
+
 ## 0.1.18 - 2018-05-23
 * Suport User Account Authentication [#33](https://github.com/treasure-data/embulk-input-google_analytics/pull/33)
 * Ignore last_record_time if non-incremental task [#34](https://github.com/treasure-data/embulk-input-google_analytics/pull/34)

data/README.md

@@ -23,7 +23,7 @@ Embulk input plugin for Google Analytics reports.
 - **retry_initial_wait_sec**: Wait seconds for exponential backoff initial value (integer, default: 2)
 
 ### **New update from verions  0.1.18**
-Started from version 0.1.18, the Plugin also supports User Account Authentication along with Service Account Authentication see: [OAuth 2.0 for Server-side Web Application](https://developers.google.com/identity/protocols/OAuth2WebServer). Extra optional configuration keys were added and the **json_key_content** is made optional 
+Started from version 0.1.18, the Plugin also supports User Account Authentication along with Service Account Authentication see: [OAuth 2.0 for Server-side Web Application](https://developers.google.com/identity/protocols/OAuth2WebServer). Extra optional configuration keys were added and the **json_key_content** is made optional
  - **client_id**: client_id for application (string, optional)
  - **client_secret**: client_secret for application (string, optional)
  - **refresh_token**: the refresh_token obtained during exchange authentication code (string, optional)
@@ -45,13 +45,13 @@ You need a service account on Google.
 select a project.</li>
   <li>Click <b>Create service account</b>.</li>
   <li>
-    
+
     In the <b>Create service account</b> window, type a name for the service
     account, and select <b>Furnish a new private key</b>. If you want to
     <a href="https://developers.google.com/identity/protocols/OAuth2ServiceAccount#delegatingauthority">grant
     Google Apps domain-wide authority</a> to the service account, also select
     <b>Enable Google Apps Domain-wide Delegation</b>.
-    
+
     Then click <b>Create</b>.</li>
 </ol>
 From: <https://developers.google.com/identity/protocols/OAuth2ServiceAccount>
@@ -89,7 +89,7 @@ in:
     }
   view_id: 123111111
   time_series: "ga:dateHour" # hourly basis
- 
+
   # https://developers.google.com/analytics/devguides/reporting/core/dimsmets
   dimensions:
     - "ga:browser"
@@ -123,5 +123,5 @@ in:
 ## Build
 
 ```
-$ rake
+$ rake build
 ```

data/Rakefile

@@ -18,4 +18,3 @@ GemReleaseHelper::Tasks.install({
   gemspec: "./embulk-input-google_analytics.gemspec",
   github_name: "treasure-data/embulk-input-google_analytics",
 })
-

data/embulk-input-google_analytics.gemspec

@@ -1,7 +1,7 @@
 
 Gem::Specification.new do |spec|
   spec.name          = "embulk-input-google_analytics"
-	spec.version       = "0.1.18"
+	spec.version       = "0.1.24"
   spec.authors       = ["uu59"]
   spec.summary       = "Google Analytics input plugin for Embulk"
   spec.description   = "Loads records from Google Analytics."
@@ -14,17 +14,25 @@ Gem::Specification.new do |spec|
   spec.require_paths = ["lib"]
 
   spec.add_dependency "httpclient"
-  spec.add_dependency "google-api-client", "0.10.1"
-  spec.add_dependency "signet"
-  spec.add_dependency "activesupport" # for Time.zone.parse, Time.zone.now
+  spec.add_dependency "google-api-client", [">= 0.11", "< 0.33.0"]
+  # signet version > 11.0 requires Ruby version >= 2.4
+  # activesupport version > 5.2.3 requires Ruby version >= 2.5
+  # Current embulk version 0.9.19 runs under jRuby 9.1.x (which is compatible with Ruby 2.3)
+  # So decide to lock these gem versions to prevent incompatible Ruby version
+  spec.add_dependency "signet", ['~> 0.7', "< 0.11.0"]
+  spec.add_dependency "activesupport", "<= 5.2.3" # for Time.zone.parse, Time.zone.now
+
   spec.add_dependency "perfect_retry", "~> 0.5"
 
   spec.add_development_dependency 'embulk', ['>= 0.8.9']
   spec.add_development_dependency 'bundler', ['>= 1.10.6']
   spec.add_development_dependency 'rake', ['>= 10.0']
-  spec.add_development_dependency 'test-unit'
+  spec.add_development_dependency 'test-unit', ['< 3.2']
   spec.add_development_dependency 'test-unit-rr'
-  spec.add_development_dependency 'simplecov'
+  # Lock simple cov and simplecov-html to prevent downloaded newer version which require ruby >= 2.4
+  # Current embulk version 0.9.19 runs under jRuby 9.1.x (which is compatible with Ruby 2.3)
+  spec.add_development_dependency 'simplecov', ['<= 0.12.0']
+  spec.add_development_dependency 'simplecov-html', ['<= 0.12.0']
   spec.add_development_dependency "codeclimate-test-reporter"
   spec.add_development_dependency "pry"
   spec.add_development_dependency "gem_release_helper", "~> 1.0"

data/lib/embulk/input/google_analytics/client.rb

@@ -3,10 +3,6 @@ require "active_support/core_ext/time"
 require "google/apis/analyticsreporting_v4"
 require "google/apis/analytics_v3"
 
-# Avoid such error:
-# PKIX path building failed: sun.security.provider.certpath.SunCertPathBuilderException: unable to find valid certification path to requested target
-Google::Apis::ClientOptions.default.use_net_http = true
-
 module Embulk
   module Input
     module GoogleAnalytics

data/lib/embulk/input/google_analytics/plugin.rb

@@ -20,6 +20,10 @@ module Embulk
             unless task['client_id'] && task['client_secret'] && task['refresh_token']
               raise ConfigError.new("client_id, client_secret and refresh_token are required when using Oauth authentication")
             end
+          elsif task['auth_method'] == Plugin::AUTH_TYPE_JSON_KEY
+            if !valid_json?(task["json_key_content"])
+              raise ConfigError.new("json_key_content is not a valid JSON object")
+            end
           end
 
           columns_list = Client.new(task).get_columns_list
@@ -106,8 +110,21 @@ module Embulk
           return {}
         end
 
+        def self.valid_json?(json_object)
+          # 'null' string is a valid string for parse function
+          # However in our case, json_content_key could not be 'null' therefore this check is added
+          if json_object == "null"
+              return false
+          end
+          begin
+            JSON.parse(json_object)
+                return true
+          rescue JSON::ParserError => e
+              return false
+          end
+        end
+
         def init
-          # PLT-6753
           if task["start_date"] && !task["end_date"]
             task["end_date"] = "today"
           end
@@ -117,8 +134,7 @@ module Embulk
           client = Client.new(task, preview?)
           columns = self.class.columns_from_task(task) + ["view_id"]
 
-          last_record_time = task['incremental'] && task["last_record_time"] ? Time.parse(task["last_record_time"]) : nil
-
+          last_record_time = Time.parse(task["last_record_time"]) if task['incremental'] && !task["last_record_time"].blank?
           latest_time_series = nil
           skip_counter, total_counter = 0, 0
           client.each_report_row do |row|
@@ -198,11 +214,13 @@ module Embulk
             # no records fetched, don't modify config_diff
             task_report = {
               start_date: task["start_date"],
-              end_date: task["end_date"],
-              last_record_time: task["last_record_time"],
+              end_date: task["end_date"]
             }
+            # write last_record_time only when last_record_time is not nil and not empty
+            unless task["last_record_time"].blank?
+              task_report[:last_record_time] = task["last_record_time"]
+            end 
           end
-
           task_report
         end
       end

data/test/embulk/input/google_analytics/test_plugin.rb

@@ -107,6 +107,26 @@ module Embulk
               end
             end
 
+            test "invalid json_key_content params for oauth" do
+              conf = valid_config["in"]
+              # Empty json_key_content
+              conf["json_key_content"] = ""
+              message = "json_key_content is not a valid JSON object"
+              assert_raise(Embulk::ConfigError.new(message)) do
+                Plugin.transaction(embulk_config(conf))
+              end
+              # null json_key_content
+              conf["json_key_content"] = "null"
+              assert_raise(Embulk::ConfigError.new(message)) do
+                Plugin.transaction(embulk_config(conf))
+              end
+              # Not a string json_key_content
+              conf["json_key_content"] = nil
+              assert_raise(Embulk::ConfigError.new(message)) do
+                Plugin.transaction(embulk_config(conf))
+              end
+            end
+
             def unknown_auth_method_message
               "Unknown Authentication method ''."
             end
@@ -360,13 +380,22 @@ module Embulk
             end
 
             sub_test_case "no records fetched" do
-              test "config_diff won't modify" do
-                plugin = Plugin.new(config, nil, nil, @page_builder)
+              data do
+              [
+                ["last_record_time is nil", [nil, nil]],
+                ["last_record_time is empty", ["", nil]],
+                ["last_record_time is blank", ["   ", nil]],
+                ["last_record_time is valid", ["2019-01-01", "2019-01-01"]]
+              ]
+              end
+              test "test calculate_next_times" do |(in_last_record_time, expected_last_record_time)|
+                config["last_record_time"] = in_last_record_time
                 expected = {
                   start_date: task["start_date"],
-                  end_date: task["end_date"],
-                  last_record_time: task["last_record_time"],
+                  end_date: task["end_date"]
                 }
+                expected[:last_record_time] = expected_last_record_time unless expected_last_record_time.blank?
+                plugin = Plugin.new(config, nil, nil, @page_builder)
                 assert_equal expected, plugin.calculate_next_times(DEFAULT_TIMEZONE, nil)
               end
             end
@@ -432,6 +461,7 @@ module Embulk
             setup do
               conf = valid_config["in"]
               conf["time_series"] = "ga:date"
+              conf["last_record_time"] = "2019-01-01"
               @config = embulk_config(conf)
             end
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk-input-google_analytics
 version: !ruby/object:Gem::Version
-  version: 0.1.18
+  version: 0.1.24
 platform: ruby
 authors:
 - uu59
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2018-05-23 00:00:00.000000000 Z
+date: 2020-08-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -27,45 +27,57 @@ dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '='
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0.11'
+    - - "<"
       - !ruby/object:Gem::Version
-        version: 0.10.1
+        version: 0.33.0
   name: google-api-client
   prerelease: false
   type: :runtime
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '='
+    - - ">="
       - !ruby/object:Gem::Version
-        version: 0.10.1
+        version: '0.11'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: 0.33.0
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '0.7'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: 0.11.0
   name: signet
   prerelease: false
   type: :runtime
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '0.7'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: 0.11.0
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "<="
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 5.2.3
   name: activesupport
   prerelease: false
   type: :runtime
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "<="
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 5.2.3
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
@@ -125,17 +137,17 @@ dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "<"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '3.2'
   name: test-unit
   prerelease: false
   type: :development
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "<"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '3.2'
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
@@ -153,17 +165,31 @@ dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "<="
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 0.12.0
   name: simplecov
   prerelease: false
   type: :development
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "<="
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 0.12.0
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "<="
+      - !ruby/object:Gem::Version
+        version: 0.12.0
+  name: simplecov-html
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "<="
+      - !ruby/object:Gem::Version
+        version: 0.12.0
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
@@ -213,6 +239,7 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
+- ".circleci/config.yml"
 - ".gitignore"
 - ".ruby-version"
 - CHANGELOG.md
@@ -220,7 +247,6 @@ files:
 - LICENSE.txt
 - README.md
 - Rakefile
-- circle.yml
 - embulk-input-google_analytics.gemspec
 - lib/embulk/input/google_analytics.rb
 - lib/embulk/input/google_analytics/client.rb

data/circle.yml

@@ -1,26 +0,0 @@
-machine:
-  java:
-    version: oraclejdk8
-  ruby:
-    version: jruby-9.1.5.0
-  environment:
-    JRUBY_OPTS: "-Xcli.debug=true --debug"
-
-dependencies:
-  pre:
-    - ruby -v
-    - bundle -v
-    - grep bundler <<< "$(gem list)" || gem i bundler --no-document
-    - bundle -v
-
-test:
-  override:
-    - bundle exec rake cov
-notify:
-  webhooks:
-    - url: http://td-beda.herokuapp.com/circleci_callback
-    
-general:
-  artifacts:
-    - "coverage"
-