embulk-input-google_analytics 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: f461f4cc05a23ba6ed9b53e23c826fc69b7c82c4
+  data.tar.gz: d73dea5ede7fb7ce0b348409808e2a7036392fa7
+SHA512:
+  metadata.gz: 5d014c2caf49c2b73c325433cb55580581184634b1a4b7dbff9d7030278f8b569b2bcf5d934e0638f92b8114ac3a0f6e1dd8df7fff0747ff4d51ab855af534ec
+  data.tar.gz: 3f36eed0bb7b8926d7aeecb7902eb0631e6a31f86e91e81f041b2e97a5894abb1a9bc47329b1acd9d96d80ed482a6915dfce4561fdb406ad37b2c6ed582d8d42

data/.gitignore

@@ -0,0 +1,6 @@
+*~
+/pkg/
+/tmp/
+/.bundle/
+/Gemfile.lock
+/coverage/

data/.ruby-version

@@ -0,0 +1 @@
+jruby-9.0.4.0

data/CHANGELOG.md

@@ -0,0 +1,3 @@
+## 0.1.0 - 2016-07-07
+
+The first release!!

data/Gemfile

@@ -0,0 +1,2 @@
+source 'https://rubygems.org/'
+gemspec

data/LICENSE.txt

@@ -0,0 +1,21 @@
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,58 @@
+[![CircleCI](https://circleci.com/gh/treasure-data/embulk-input-google_analytics/tree/master.svg?style=svg)](https://circleci.com/gh/treasure-data/embulk-input-google_analytics/tree/master)
+[![Code Climate](https://codeclimate.com/github/treasure-data/embulk-input-google_analytics/badges/gpa.svg)](https://codeclimate.com/github/treasure-data/embulk-input-google_analytics)
+[![Test Coverage](https://codeclimate.com/github/treasure-data/embulk-input-google_analytics/badges/coverage.svg)](https://codeclimate.com/github/treasure-data/embulk-input-google_analytics/coverage)
+[![Issue Count](https://codeclimate.com/github/treasure-data/embulk-input-google_analytics/badges/issue_count.svg)](https://codeclimate.com/github/treasure-data/embulk-input-google_analytics)
+[![Gem Version](https://badge.fury.io/rb/embulk-input-google_analytics.svg)](https://badge.fury.io/rb/embulk-input-google_analytics)
+
+# Google Analytics input plugin for Embulk
+
+Embulk input plugin for Google Analytics reports.
+
+## Configuration
+
+- **json_key_content**: See example config.
+- **view_id**: View ID for target data. You can find it on [Google Analytics page (you need a permission to access Admin page](https://lucidpress.zendesk.com/hc/en-us/articles/207335356-Find-your-Google-Analytics-Tracking-ID-View-ID) (string, required)
+- **time_series**: Only `ga:dateHour` or `ga:date` (string, required)
+- **dimensions**: Target dimensions (array, default: `[]` )
+- **metrics**: Target metrics (array, default: `[]` )
+- **start_date**: Target report start date (string, default: [7 days ago](https://developers.google.com/analytics/devguides/reporting/core/v4/rest/v4/reports/batchGet#reportrequest))
+- **end_date**: Target report end date (string, default: [1 day ago](https://developers.google.com/analytics/devguides/reporting/core/v4/rest/v4/reports/batchGet#reportrequest))
+
+## Example
+
+```yaml
+in:
+  type: google_analytics
+  json_key_content: |
+    {
+      "type": "service_account",
+      "project_id": "....",
+      "private_key_id": "....",
+      "private_key": "-----BEGIN PRIVATE KEY-----\n..........................\n-----END PRIVATE KEY-----\n",
+      "client_email": ".....",
+      "client_id": ".........",
+      "auth_uri": "https://accounts.google.com/o/oauth2/auth",
+      "token_uri": "https://accounts.google.com/o/oauth2/token",
+      "auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs",
+      "client_x509_cert_url": ".........."
+    }
+  view_id: 123111111
+  time_series: "ga:dateHour" # hourly basis
+ 
+  # https://developers.google.com/analytics/devguides/reporting/core/dimsmets
+  dimensions:
+    - "ga:browser"
+  metrics:
+    - "ga:visits"
+    - "ga:pageviews"
+
+  start_date: "2016-06-27"
+  end_date: "2016-06-28"
+```
+
+
+## Build
+
+```
+$ rake
+```

data/Rakefile

@@ -0,0 +1,15 @@
+require "bundler/gem_tasks"
+
+task default: :test
+
+desc "Run tests"
+task :test do
+  ruby("--debug", "test/run-test.rb", "--use-color=yes", "--collector=dir")
+end
+
+desc "Run tests with coverage"
+task :cov do
+  ENV["COVERAGE"] = "1"
+  ruby("--debug", "test/run-test.rb", "--use-color=yes", "--collector=dir")
+end
+

data/circle.yml

@@ -0,0 +1,23 @@
+machine:
+  java:
+    version: oraclejdk8
+  ruby:
+    version: jruby-9.0.4.0
+  environment:
+    JRUBY_OPTS: "-Xcli.debug=true --debug"
+
+dependencies:
+  pre:
+    - ruby -v
+    - bundle -v
+    - grep bundler <<< "$(gem list)" || gem i bundler --no-document
+    - bundle -v
+
+test:
+  override:
+    - bundle exec rake cov
+
+general:
+  artifacts:
+    - "coverage"
+

data/embulk-input-google_analytics.gemspec

@@ -0,0 +1,29 @@
+
+Gem::Specification.new do |spec|
+  spec.name          = "embulk-input-google_analytics"
+  spec.version       = "0.1.0"
+  spec.authors       = ["uu59"]
+  spec.summary       = "Google Analytics input plugin for Embulk"
+  spec.description   = "Loads records from Google Analytics."
+  spec.email         = ["k@uu59.org"]
+  spec.licenses      = ["MIT"]
+  spec.homepage      = "https://github.com/treasure-data/embulk-input-google_analytics"
+
+  spec.files         = `git ls-files`.split("\n") + Dir["classpath/*.jar"]
+  spec.test_files    = spec.files.grep(%r{^(test|spec)/})
+  spec.require_paths = ["lib"]
+
+  spec.add_dependency "httpclient"
+  spec.add_dependency "google-api-client", "~> 0.9"
+  spec.add_dependency "signet"
+  spec.add_dependency "activesupport" # for Time.zone.parse
+
+  spec.add_development_dependency 'embulk', ['>= 0.8.9']
+  spec.add_development_dependency 'bundler', ['>= 1.10.6']
+  spec.add_development_dependency 'rake', ['>= 10.0']
+  spec.add_development_dependency 'test-unit'
+  spec.add_development_dependency 'test-unit-rr'
+  spec.add_development_dependency 'simplecov'
+  spec.add_development_dependency "codeclimate-test-reporter"
+  spec.add_development_dependency "pry"
+end

data/lib/embulk/input/google_analytics/client.rb

@@ -0,0 +1,160 @@
+require "active_support/core_ext/time"
+require "google/apis/analyticsreporting_v4"
+require "google/apis/analytics_v3"
+
+module Embulk
+  module Input
+    module GoogleAnalytics
+      class Client
+        attr_reader :task
+
+        def initialize(task, is_preview = false)
+          @task = task
+          @is_preview = is_preview
+        end
+
+        def preview?
+          @is_preview
+        end
+
+        def each_report_row(&block)
+          page_token = nil
+          Embulk.logger.info "view_id:#{view_id} timezone has been set as '#{get_profile[:timezone]}'"
+
+          loop do
+            result = get_reports(page_token)
+            report = result.to_h[:reports].first
+
+            unless page_token
+              # display for first request only
+              Embulk.logger.info "Total: #{report[:data][:row_count]} rows. Fetched first response"
+            end
+
+            if !report[:data].has_key?(:rows)
+              Embulk.logger.warn "Result doesn't contain rows."
+              break
+            end
+
+            if report[:data][:rows].empty?
+              Embulk.logger.warn "Result has 0 rows."
+              break
+            end
+
+            dimensions = report[:column_header][:dimensions]
+            metrics = report[:column_header][:metric_header][:metric_header_entries].map{|m| m[:name]}
+            report[:data][:rows].each do |row|
+              dim = dimensions.zip(row[:dimensions]).to_h
+              met = metrics.zip(row[:metrics].first[:values]).to_h
+              format_row = dim.merge(met)
+              time = format_row[task["time_series"]]
+              format_row[task["time_series"]] = time_parse_with_profile_timezone(time)
+              block.call format_row
+            end
+
+            break if preview?
+
+            unless page_token = report[:next_page_token]
+              break
+            end
+            Embulk.logger.info "Fetching report with page_token: #{page_token}"
+          end
+        end
+
+        def get_profile
+          @profile ||=
+            begin
+              profile = get_all_profiles.to_h[:items].find do |prof|
+                prof[:id] == view_id
+              end
+
+              unless profile
+                raise Embulk::ConfigError.new("Can't find view_id:#{view_id} profile via Google Analytics API.")
+              end
+
+              profile
+            end
+        end
+
+        def get_all_profiles
+          service = Google::Apis::AnalyticsV3::AnalyticsService.new
+          service.authorization = auth
+
+          Embulk.logger.debug "Fetching profile from API"
+          service.list_profiles("~all", "~all")
+        end
+
+        def time_parse_with_profile_timezone(time_string)
+          date_format =
+            case task["time_series"]
+            when "ga:dateHour"
+              "%Y%m%d%H"
+            when "ga:date"
+              "%Y%m%d"
+            end
+          parts = Date._strptime(time_string, date_format)
+
+          orig_timezone = Time.zone
+          Time.zone = get_profile[:timezone]
+          Time.zone.local(*parts.values_at(:year, :mon, :mday, :hour)).to_time
+        ensure
+          Time.zone = orig_timezone
+        end
+
+        def get_reports(page_token = nil)
+          # https://developers.google.com/analytics/devguides/reporting/core/v4/rest/v4/reports/batchGet
+          service = Google::Apis::AnalyticsreportingV4::AnalyticsReportingService.new
+          service.authorization = auth
+
+          request = Google::Apis::AnalyticsreportingV4::GetReportsRequest.new
+          request.report_requests = build_report_request(page_token)
+
+          Embulk.logger.info "Query to Core Report API: #{request.to_json}"
+          service.batch_get_reports request
+        end
+
+        def get_columns_list
+          # https://developers.google.com/analytics/devguides/reporting/metadata/v3/reference/metadata/columns/list
+          service = Google::Apis::AnalyticsV3::AnalyticsService.new
+          service.authorization = auth
+          service.list_metadata_columns("ga").to_h[:items]
+        end
+
+        def build_report_request(page_token = nil)
+          query = {
+            view_id: view_id,
+            dimensions: [{name: task["time_series"]}] + task["dimensions"].map{|d| {name: d}},
+            metrics: task["metrics"].map{|m| {expression: m}},
+            include_empty_rows: true,
+            page_size: preview? ? 10 : 10000,
+          }
+
+          if task["start_date"] || task["end_date"]
+            query[:date_ranges] = [{
+              start_date: task["start_date"],
+              end_date: task["end_date"],
+            }]
+          end
+
+          if page_token
+            query[:page_token] = page_token
+          end
+
+          [query]
+        end
+
+        def view_id
+          task["view_id"]
+        end
+
+        def auth
+          Google::Auth::ServiceAccountCredentials.make_creds(
+            json_key_io: StringIO.new(task["json_key_content"]),
+            scope: "https://www.googleapis.com/auth/analytics.readonly"
+          )
+        rescue => e
+          raise ConfigError.new(e.message)
+        end
+      end
+    end
+  end
+end

data/lib/embulk/input/google_analytics/plugin.rb

@@ -0,0 +1,101 @@
+module Embulk
+  module Input
+    module GoogleAnalytics
+      class Plugin < InputPlugin
+        ::Embulk::Plugin.register_input("google_analytics", self)
+
+        # https://developers.google.com/analytics/devguides/reporting/core/dimsmets
+
+        def self.transaction(config, &control)
+          task = task_from_config(config)
+          unless %w(ga:date ga:dateHour).include?(task["time_series"])
+            raise ConfigError.new("Unknown time_series '#{task["time_series"]}'. Use 'ga:dateHour' or 'ga:date'")
+          end
+          columns_list = Client.new(task).get_columns_list
+
+          columns = columns_from_task(task).map do |col_name|
+            col_info = columns_list.find{|col| col[:id] == col_name}
+            raise ConfigError.new("Unknown metric/dimension '#{col_name}'") unless col_info
+
+            col_type = 
+              case col_info[:attributes][:dataType]
+              when "STRING"
+                :string
+              when "INTEGER"
+                :long
+              when "PERCENT", "FLOAT", "CURRENCY"
+                :double
+              when "TIME"
+                :timestamp
+              end
+
+            # time_series column should be timestamp
+            if col_name == task["time_series"]
+              col_type = :timestamp
+            end
+            Column.new(nil, canonicalize_column_name(col_name), col_type)
+          end
+
+          resume(task, columns, 1, &control)
+        end
+
+        def self.resume(task, columns, count, &control)
+          task_reports = yield(task, columns, count)
+
+          next_config_diff = {}
+          return next_config_diff
+        end
+
+        def self.task_from_config(config)
+          json_key_content = config.param("json_key_content", :string)
+          {
+            "json_key_content" => json_key_content,
+            "view_id" => config.param("view_id", :string),
+            "dimensions" => config.param("dimensions", :array, default: []),
+            "metrics" => config.param("metrics", :array, default: []),
+            "time_series" => config.param("time_series", :string),
+            "start_date" => config.param("start_date", :string, default: nil),
+            "end_date" => config.param("end_date", :string, default: nil),
+          }
+        end
+
+        def self.columns_from_task(task)
+          [
+            task["time_series"],
+            task["dimensions"],
+            task["metrics"],
+          ].flatten.uniq
+        end
+
+        def self.canonicalize_column_name(name)
+          # ga:dateHour -> date_hour
+          name.gsub(/^ga:/, "").gsub(/[A-Z]+/, "_\\0").gsub(/^_/, "").downcase
+        end
+
+        def init
+        end
+
+        def run
+          client = Client.new(task, preview?)
+          columns = self.class.columns_from_task(task)
+
+          client.each_report_row do |row|
+            values = row.values_at(*columns)
+            page_builder.add values
+          end
+          page_builder.finish
+
+          task_report = {}
+          return task_report
+        end
+
+        def preview?
+          org.embulk.spi.Exec.isPreview()
+        rescue java.lang.NullPointerException
+          false
+        end
+
+      end
+    end
+  end
+end

data/lib/embulk/input/google_analytics.rb

@@ -0,0 +1,9 @@
+require "embulk/input/google_analytics/client"
+require "embulk/input/google_analytics/plugin"
+
+module Embulk
+  module Input
+    module GoogleAnalytics
+    end
+  end
+end

data/test/embulk/input/google_analytics/test_client.rb

@@ -0,0 +1,290 @@
+require "embulk"
+Embulk.setup
+
+require "embulk/input/google_analytics"
+require "override_assert_raise"
+require "fixture_helper"
+
+module Embulk
+  module Input
+    module GoogleAnalytics
+      class TestClient < Test::Unit::TestCase
+        include OverrideAssertRaise
+        include FixtureHelper
+
+        sub_test_case "get_profile" do
+          setup do
+            conf = valid_config["in"]
+            @task = task(embulk_config(conf))
+            @client = Client.new(@task)
+          end
+
+          test "find view_id profile" do
+            target_profile = {
+              id: @task["view_id"],
+            }
+
+            mock(@client).get_all_profiles do
+              {
+                items: [
+                  { id: 1 },
+                  target_profile,
+                  { id: 2 },
+                ]
+              }
+            end
+
+            assert_equal target_profile, @client.get_profile
+          end
+
+          test "raise ConfigError when view_id is not found" do
+            mock(@client).get_all_profiles do
+              {
+                items: [
+                  { id: 1 },
+                  { id: 2 },
+                ]
+              }
+            end
+
+            assert_raise(Embulk::ConfigError) do
+              @client.get_profile
+            end
+          end
+        end
+
+        sub_test_case "build_report_request" do
+          setup do
+            conf = valid_config["in"]
+            @task = task(embulk_config(conf))
+            @client = Client.new(@task)
+          end
+
+          test "page_token = nil" do
+            req = @client.build_report_request
+            expected = [
+              {
+                view_id: "101111111",
+                dimensions: [
+                  {name: "ga:dateHour"}, {name: "ga:browser"}
+                ],
+                metrics: [
+                  {expression: "ga:visits"}, {expression: "ga:pageviews"}
+                ],
+                include_empty_rows: true,
+                page_size: 10000
+              }
+            ]
+            assert_equal expected, req
+          end
+
+          test "page_token = 123" do
+            req = @client.build_report_request(123)
+            expected = [
+              {
+                view_id: "101111111",
+                dimensions: [
+                  {name: "ga:dateHour"}, {name: "ga:browser"}
+                ],
+                metrics: [
+                  {expression: "ga:visits"}, {expression: "ga:pageviews"}
+                ],
+                include_empty_rows: true,
+                page_size: 10000,
+                page_token: 123
+              }
+            ]
+            assert_equal expected, req
+          end
+
+          test "date range given" do
+            conf = valid_config["in"]
+            conf["start_date"] = "2000-01-01"
+            conf["end_date"] = "2000-01-07"
+            task = task(embulk_config(conf))
+            client = Client.new(task)
+            req = client.build_report_request
+
+            expected = [
+              {
+                view_id: "101111111",
+                dimensions: [
+                  {name: "ga:dateHour"}, {name: "ga:browser"}
+                ],
+                metrics: [
+                  {expression: "ga:visits"}, {expression: "ga:pageviews"}
+                ],
+                include_empty_rows: true,
+                page_size: 10000,
+                date_ranges: [
+                  {
+                    start_date: conf["start_date"],
+                    end_date: conf["end_date"],
+                  }
+                ]
+              }
+            ]
+            assert_equal expected, req
+          end
+        end
+
+        sub_test_case "time_parse_with_profile_timezone" do
+          setup do
+            conf = valid_config["in"]
+            conf["time_series"] = time_series
+            @client = Client.new(task(embulk_config(conf)))
+          end
+
+          sub_test_case "dateHour" do
+            setup do
+              stub(@client).get_profile { {timezone: "America/Los_Angeles" } }
+            end
+
+            test "in dst" do
+              time = @client.time_parse_with_profile_timezone("2016060122")
+              assert_equal Time.parse("2016-06-01 22:00:00 -07:00"), time
+            end
+
+            test "not in dst" do
+              time = @client.time_parse_with_profile_timezone("2016010122")
+              assert_equal Time.parse("2016-01-01 22:00:00 -08:00"), time
+            end
+
+            def time_series
+              "ga:dateHour"
+            end
+          end
+
+          sub_test_case "date" do
+            setup do
+              stub(@client).get_profile { {timezone: "America/Los_Angeles" } }
+            end
+
+            test "in dst" do
+              time = @client.time_parse_with_profile_timezone("20160601")
+              assert_equal Time.parse("2016-06-01 00:00:00 PDT"), time
+            end
+
+            test "not in dst" do
+              time = @client.time_parse_with_profile_timezone("2016010122")
+              assert_equal Time.parse("2016-01-01 00:00:00 PST"), time
+            end
+
+            def time_series
+              "ga:date"
+            end
+          end
+        end
+
+        sub_test_case "auth" do
+          setup do
+            conf = valid_config["in"]
+            @client = Client.new(task(embulk_config(conf)))
+          end
+
+          test "raise ConfigError when auth failed" do
+            stub(Google::Auth::ServiceAccountCredentials).make_creds { raise "some error" }
+            assert_raise(Embulk::ConfigError) do
+              @client.auth
+            end
+          end
+        end
+
+        sub_test_case "each_report_row" do
+          setup do
+            conf = valid_config["in"]
+            @client = Client.new(task(embulk_config(conf)))
+            stub(@client).get_profile { {timezone: "Asia/Tokyo"} }
+            @logger = Logger.new(File::NULL)
+            stub(Embulk).logger { @logger }
+          end
+
+          test "without pagination" do
+            stub(@client).get_reports { report }
+            fetched_rows = []
+            @client.each_report_row do |row|
+              fetched_rows << row
+            end
+
+            expected = [
+              {
+                "ga:dateHour" => @client.time_parse_with_profile_timezone("2016060120"),
+                "ga:browser" => "curl",
+                "ga:visits" => "1",
+                "ga:pageviews" => "1",
+              },
+              {
+                "ga:dateHour" => @client.time_parse_with_profile_timezone("2016060121"),
+                "ga:browser" => "curl",
+                "ga:visits" => "2",
+                "ga:pageviews" => "2",
+              },
+              {
+                "ga:dateHour" => @client.time_parse_with_profile_timezone("2016060122"),
+                "ga:browser" => "curl",
+                "ga:visits" => "3",
+                "ga:pageviews" => "3",
+              },
+            ]
+            assert_equal expected, fetched_rows
+          end
+
+          test "with pagination" do
+            next_page_token = "10000"
+            mock(@client).get_reports(nil) { report_with_pages }
+            mock(@client).get_reports(next_page_token) { report }
+            fetched_rows = []
+            @client.each_report_row do |row|
+              fetched_rows << row
+            end
+            assert_equal 6, fetched_rows.length
+          end
+
+          sub_test_case "logger" do
+            test "with empty rows" do
+              response = report.dup
+              response[:reports].first[:data][:rows] = []
+              response[:reports].first[:data][:row_count] = 0
+              stub(@client).get_reports { response }
+
+              mock(@logger).warn("Result has 0 rows.")
+              @client.each_report_row {}
+            end
+
+            test "without rows" do
+              response = report.dup
+              response[:reports].first[:data].delete(:rows)
+              stub(@client).get_reports { response }
+
+              mock(@logger).warn("Result doesn't contain rows.")
+              @client.each_report_row {}
+            end
+          end
+
+          def report_with_pages
+            response = report.dup
+            response[:reports].first[:next_page_token] = "10000"
+            response
+          end
+
+          def report
+            json = fixture_read("reports.json")
+            JSON.parse(json, symbolize_names: true)
+          end
+        end
+
+        def task(config)
+          Plugin.task_from_config(config)
+        end
+
+        def valid_config
+          fixture_load("valid.yml")
+        end
+
+        def embulk_config(hash)
+          Embulk::DataSource.new(hash)
+        end
+      end
+    end
+  end
+end

data/test/embulk/input/google_analytics/test_plugin.rb

@@ -0,0 +1,214 @@
+require "embulk"
+Embulk.setup
+
+require "embulk/input/google_analytics"
+require "override_assert_raise"
+require "fixture_helper"
+
+module Embulk
+  module Input
+    module GoogleAnalytics
+      class TestPlugin < Test::Unit::TestCase
+        include OverrideAssertRaise
+        include FixtureHelper
+
+        sub_test_case ".transaction" do
+          setup do
+            any_instance_of(Client) do |klass|
+              stub(klass).get_columns_list do
+                [
+                  {id: "ga:dateHour", attributes: {dataType: "STRING"}},
+                  {id: "ga:date", attributes: {dataType: "STRING"}},
+                  {id: "ga:browser", attributes: {dataType: "STRING"}},
+                  {id: "ga:visits", attributes: {dataType: "INTEGER"}},
+                  {id: "ga:pageviews", attributes: {dataType: "INTEGER"}},
+                ]
+              end
+            end
+          end
+
+          test "not raised exception" do
+            stub(Plugin).resume { Hash.new }
+            assert_nothing_raised do
+              Plugin.transaction(embulk_config(valid_config["in"]))
+            end
+          end
+
+          test "assemble expected columns" do
+            columns = [
+              Embulk::Column.new(nil, "date_hour", :timestamp),
+              Embulk::Column.new(nil, "browser", :string),
+              Embulk::Column.new(nil, "visits", :long),
+              Embulk::Column.new(nil, "pageviews", :long),
+            ]
+            mock(Plugin).resume(anything, columns, 1)
+            Plugin.transaction(embulk_config(valid_config["in"]))
+          end
+
+          sub_test_case "raise error when unknown column given" do
+            setup { stub(Plugin).resume { Hash.new } }
+
+            test "for dimensions" do
+              conf = valid_config["in"]
+              conf["dimensions"] << unknown_col_name
+              assert_raise(Embulk::ConfigError.new(expected_message)) do
+                Plugin.transaction(embulk_config(conf))
+              end
+            end
+
+            test "for metrics" do
+              conf = valid_config["in"]
+              conf["metrics"] << unknown_col_name
+              assert_raise(Embulk::ConfigError.new(expected_message)) do
+                Plugin.transaction(embulk_config(conf))
+              end
+            end
+
+            test "for time_series" do
+              conf = valid_config["in"]
+              conf["time_series"] = unknown_col_name
+              message = "Unknown time_series 'ga:foooooo'. Use 'ga:dateHour' or 'ga:date'"
+              assert_raise(Embulk::ConfigError.new(message)) do
+                Plugin.transaction(embulk_config(conf))
+              end
+            end
+
+            def unknown_col_name
+              "ga:foooooo"
+            end
+
+            def expected_message
+              "Unknown metric/dimension '#{unknown_col_name}'"
+            end
+          end
+
+          sub_test_case "type conversion" do
+            setup do
+              any_instance_of(Client) do |klass|
+                stub(klass).get_columns_list do
+                  [
+                    {id: "ga:dateHour", attributes: {dataType: "STRING"}},
+                    {id: "ga:itemsPerPurchase", attributes: {dataType: "FLOAT"}},
+                    {id: "ga:visits", attributes: {dataType: "INTEGER"}},
+                    {id: "ga:CPM", attributes: {dataType: "CURRENCY"}},
+                    {id: "ga:CTR", attributes: {dataType: "PERCENT"}},
+                    {id: "ga:sessionDuration", attributes: {dataType: "TIME"}},
+                  ]
+                end
+              end
+            end
+
+            test "Convert Embulk data types" do
+              conf = valid_config["in"]
+              conf["dimensions"] = []
+              conf["metrics"] = [
+                "ga:sessionDuration",
+                "ga:CPM",
+                "ga:CTR",
+                "ga:visits",
+                "ga:itemsPerPurchase",
+              ]
+              expected_columns = [
+                Column.new(nil, "date_hour", :timestamp),
+                Column.new(nil, "session_duration", :timestamp),
+                Column.new(nil, "cpm", :double),
+                Column.new(nil, "ctr", :double),
+                Column.new(nil, "visits", :long),
+                Column.new(nil, "items_per_purchase", :double),
+              ]
+
+              mock(Plugin).resume(anything, expected_columns, anything)
+              Plugin.transaction(embulk_config(conf))
+            end
+          end
+        end
+
+        sub_test_case ".run" do
+          sub_test_case "returned value should be added into page_builder" do
+            setup do
+              @page_builder = Object.new
+              conf = valid_config["in"]
+              conf["time_series"] = time_series
+              @plugin = Plugin.new(embulk_config(conf), nil, nil, @page_builder)
+            end
+
+            sub_test_case "time_series: 'ga:dateHour'" do
+              def time_series
+                "ga:dateHour"
+              end
+
+              test "HH:00:00 time given" do
+                Time.zone = "America/Los_Angeles"
+                time = Time.zone.parse("2016-06-01 12:00:00").to_time
+                any_instance_of(Client) do |klass|
+                  stub(klass).each_report_row do |block|
+                    row = {
+                      "ga:dateHour" => time,
+                      "ga:browser" => "wget",
+                      "ga:visits" => 3,
+                      "ga:pageviews" => 4,
+                    }
+                    block.call row
+                  end
+                end
+
+                mock(@page_builder).add([time, "wget", 3, 4])
+                mock(@page_builder).finish
+                @plugin.run
+              end
+            end
+
+            sub_test_case "time_series: 'ga:date'" do
+              def time_series
+                "ga:date"
+              end
+
+              test "00:00:00 time given" do
+                Time.zone = "America/Los_Angeles"
+                time = Time.zone.parse("2016-06-01 00:00:00").to_time
+                any_instance_of(Client) do |klass|
+                  stub(klass).each_report_row do |block|
+                    row = {
+                      "ga:date" => time,
+                      "ga:browser" => "wget",
+                      "ga:visits" => 3,
+                      "ga:pageviews" => 4,
+                    }
+                    block.call row
+                  end
+                end
+
+                mock(@page_builder).add([time, "wget", 3, 4])
+                mock(@page_builder).finish
+                @plugin.run
+              end
+            end
+          end
+        end
+
+        sub_test_case "canonicalize_column_name" do
+          data do
+            [
+              ["typical", ["ga:dateHour", "date_hour"]],
+              ["all capital", ["ga:CPM", "cpm"]],
+              ["capitals with word", ["ga:goalXXValue", "goal_xxvalue"]],
+              ["ID", ["ga:adwordsCustomerID", "adwords_customer_id"]],
+              ["word + capitals", ["ga:dcmCTR", "dcm_ctr"]],
+            ]
+          end
+          test "converting" do |(target, expected)|
+            assert_equal expected, Plugin.canonicalize_column_name(target)
+          end
+        end
+
+        def valid_config
+          fixture_load("valid.yml")
+        end
+
+        def embulk_config(hash)
+          Embulk::DataSource.new(hash)
+        end
+      end
+    end
+  end
+end

data/test/fixture_helper.rb

@@ -0,0 +1,16 @@
+require "yaml"
+require "pathname"
+
+module FixtureHelper
+  def fixture_dir
+    Pathname.new(__FILE__).dirname.join("fixtures")
+  end
+
+  def fixture_read(name)
+    fixture_dir.join(name).read
+  end
+
+  def fixture_load(name)
+    YAML.load fixture_read(name)
+  end
+end

data/test/fixtures/reports.json

@@ -0,0 +1,71 @@
+{
+  "reports": [
+    {
+      "column_header": {
+        "dimensions": [
+          "ga:dateHour",
+          "ga:browser"
+        ],
+        "metric_header": {
+          "metric_header_entries": [
+            {
+              "type": "INTEGER",
+              "name": "ga:visits"
+            },
+            {
+              "type": "INTEGER",
+              "name": "ga:pageviews"
+            }
+          ]
+        }
+      },
+      "data": {
+        "row_count": 3,
+        "rows": [
+          {
+            "metrics": [
+              {
+                "values": [
+                  "1",
+                  "1"
+                ]
+              }
+            ],
+            "dimensions": [
+              "2016060120",
+              "curl"
+            ]
+          },
+          {
+            "metrics": [
+              {
+                "values": [
+                  "2",
+                  "2"
+                ]
+              }
+            ],
+            "dimensions": [
+              "2016060121",
+              "curl"
+            ]
+          },
+          {
+            "metrics": [
+              {
+                "values": [
+                  "3",
+                  "3"
+                ]
+              }
+            ],
+            "dimensions": [
+              "2016060122",
+              "curl"
+            ]
+          }
+        ]
+      }
+    }
+  ]
+}

data/test/fixtures/valid.yml

@@ -0,0 +1,17 @@
+in:
+  type: google_analytics
+  json_key_content: |
+    {
+      "type": "service_account"
+    }
+  view_id: 101111111
+  time_series: "ga:dateHour"
+ 
+  dimensions:
+    - "ga:browser"
+  metrics:
+    - "ga:visits"
+    - "ga:pageviews"
+
+out:
+  type: stdout

data/test/override_assert_raise.rb

@@ -0,0 +1,24 @@
+module OverrideAssertRaise
+  # NOTE: Embulk 0.7.1+ required to raise ConfigError to do as `ConfigError.new("message")`,
+  #       original `assert_raise` method can't catch that, but `begin .. rescue` can.
+  #       So we override assert_raise as below.
+  def assert_raise(expected_class_or_instance = StandardError, &block)
+    begin
+      expected_class = expected_class_or_instance.is_a?(Class) ? expected_class_or_instance : expected_class_or_instance.class
+      block.call
+      assert_equal expected_class, nil
+    rescue ::Test::Unit::AssertionFailedError => e
+      # failed assert raises this Error and that extends StandardError, so rescue it first
+      raise e
+    rescue expected_class => e
+      # https://github.com/test-unit/test-unit/issues/94
+      if expected_class_or_instance == expected_class
+        assert true # passed
+      else
+        assert_equal expected_class_or_instance.message, e.message
+      end
+    rescue => e
+      assert_equal(expected_class, e.class) # not expected one raised
+    end
+  end
+end

data/test/run-test.rb

@@ -0,0 +1,25 @@
+#!/usr/bin/env ruby
+
+base_dir = File.expand_path(File.join(File.dirname(__FILE__), ".."))
+lib_dir = File.join(base_dir, "lib")
+test_dir = File.join(base_dir, "test")
+
+require "test-unit"
+require "test/unit/rr"
+
+$LOAD_PATH.unshift(lib_dir)
+$LOAD_PATH.unshift(test_dir)
+
+ENV["TEST_UNIT_MAX_DIFF_TARGET_STRING_SIZE"] ||= "5000"
+
+if ENV["COVERAGE"]
+  if ENV["CI"]
+    require "codeclimate-test-reporter"
+    CodeClimate::TestReporter.start
+  else
+    require 'simplecov'
+    SimpleCov.start 'test_frameworks'
+  end
+end
+
+exit Test::Unit::AutoRunner.run(true, test_dir)

metadata

@@ -0,0 +1,238 @@
+--- !ruby/object:Gem::Specification
+name: embulk-input-google_analytics
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- uu59
+autorequire:
+bindir: bin
+cert_chain: []
+date: 2016-07-07 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  name: httpclient
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.9'
+  name: google-api-client
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.9'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  name: signet
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  name: activesupport
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.8.9
+  name: embulk
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.8.9
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.10.6
+  name: bundler
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.10.6
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  name: rake
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '10.0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  name: test-unit
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  name: test-unit-rr
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  name: simplecov
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  name: codeclimate-test-reporter
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  name: pry
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Loads records from Google Analytics.
+email:
+- k@uu59.org
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".ruby-version"
+- CHANGELOG.md
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- circle.yml
+- embulk-input-google_analytics.gemspec
+- lib/embulk/input/google_analytics.rb
+- lib/embulk/input/google_analytics/client.rb
+- lib/embulk/input/google_analytics/plugin.rb
+- test/embulk/input/google_analytics/test_client.rb
+- test/embulk/input/google_analytics/test_plugin.rb
+- test/fixture_helper.rb
+- test/fixtures/reports.json
+- test/fixtures/valid.yml
+- test/override_assert_raise.rb
+- test/run-test.rb
+homepage: https://github.com/treasure-data/embulk-input-google_analytics
+licenses:
+- MIT
+metadata: {}
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project:
+rubygems_version: 2.4.8
+signing_key:
+specification_version: 4
+summary: Google Analytics input plugin for Embulk
+test_files:
+- test/embulk/input/google_analytics/test_client.rb
+- test/embulk/input/google_analytics/test_plugin.rb
+- test/fixture_helper.rb
+- test/fixtures/reports.json
+- test/fixtures/valid.yml
+- test/override_assert_raise.rb
+- test/run-test.rb