embulk-input-google_spreadsheets 1.0.0

data/lib/embulk/input/google_spreadsheets/typecast/strict_typecast.rb

@@ -0,0 +1,236 @@
+require 'time_with_zone'
+require_relative 'base'
+require_relative 'timestamp_format_util'
+
+module Embulk
+  module Input
+    class GoogleSpreadsheets < InputPlugin
+      module Typecast
+        class StrictTypecast < Base
+          # It has assumed that argument `value` is an instance of any of the below classes.
+          # NilClass
+          # String
+          # TrueClass
+          # FalseClass
+          # Integer
+          # Float
+          # Array
+          # Hash
+          # Time
+
+          def as_string(value)
+            return nil if null_string?(value)
+
+            case value
+            when nil
+              nil
+            when String
+              value
+            when TrueClass
+              value.to_s
+            when FalseClass
+              value.to_s
+            when Integer
+              value.to_s
+            when Float
+              value.to_s
+            when Array
+              value.to_json
+            when Hash
+              value.to_json
+            when Time
+              # TODO: support Time class
+              raise TypecastError.new "`embulk-input-google_spreadsheets`: cannot typecast Time to String: \"#{value}\""
+            else
+              raise TypecastError.new "`embulk-input-google_spreadsheets`: cannot typecast #{value.class} to String: \"#{value}\""
+            end
+          end
+
+          def as_long(value)
+            return nil if null_string?(value)
+
+            # `to_i` issue incorrect typecast, so use Integer()
+            # example
+            #   [1] pry(main)> '111:222:333'.to_i
+            #   => 111
+            #   [2] pry(main)> Integer('111:222:333')
+            #   ArgumentError: invalid value for Integer(): "111:222:333"
+            #   from (pry):2:in `Integer'
+            case value
+            when nil
+              nil
+            when String
+              return nil if value.is_a?(String) and value.empty?
+              begin
+                Integer(value)
+              rescue ArgumentError => e
+                raise TypecastError.new "`embulk-input-google_spreadsheets`: #{e}"
+              end
+            when TrueClass
+              raise TypecastError.new "`embulk-input-google_spreadsheets`: cannot typecast true to Integer: \"#{value}\""
+            when FalseClass
+              raise TypecastError.new "`embulk-input-google_spreadsheets`: cannot typecast false to Integer: \"#{value}\""
+            when Integer
+              value
+            when Float
+              raise TypecastError.new "`embulk-input-google_spreadsheets`: cannot typecast Float to Integer: \"#{value}\""
+            when Array
+              raise TypecastError.new "`embulk-input-google_spreadsheets`: cannot typecast Array to Integer: \"#{value}\""
+            when Hash
+              raise TypecastError.new "`embulk-input-google_spreadsheets`: cannot typecast Hash to Integer: \"#{value}\""
+            when Time
+              value.to_i
+            else
+              raise TypecastError.new "`embulk-input-google_spreadsheets`: cannot typecast #{value.class} to Integer: \"#{value}\""
+            end
+          end
+
+          def as_double(value)
+            return nil if null_string?(value)
+
+            # `to_f` issue incorrect typecast, so use Float()
+            # example
+            #   [1] pry(main)> '111:222:333'.to_f
+            #   => 111.0
+            #   [2] pry(main)> Float('111:222:333')
+            #   ArgumentError: invalid value for Float(): "111:222:333"
+            #   from (pry):2:in `Float'
+            case value
+            when nil
+              nil
+            when String, Integer
+              return nil if value.is_a?(String) and value.empty?
+              begin
+                Float(value)
+              rescue ArgumentError => e
+                raise TypecastError.new "`embulk-input-google_spreadsheets`: #{e}"
+              end
+            when TrueClass
+              raise TypecastError.new "`embulk-input-google_spreadsheets`: cannot typecast true to Float: \"#{value}\""
+            when FalseClass
+              raise TypecastError.new "`embulk-input-google_spreadsheets`: cannot typecast false to Float: \"#{value}\""
+              # when Integer then ...
+              # => merge with String case
+            when Float
+              value
+            when Array
+              raise TypecastError.new "`embulk-input-google_spreadsheets`: cannot typecast Array to Float: \"#{value}\""
+            when Hash
+              raise TypecastError.new "`embulk-input-google_spreadsheets`: cannot typecast Hash to Float: \"#{value}\""
+            when Time
+              value.to_f
+            else
+              raise TypecastError.new "`embulk-input-google_spreadsheets`: cannot typecast #{value.class} to Float: \"#{value}\""
+            end
+          end
+
+          BOOLEAN_TRUE_VALUES = %w(yes y true t 1)
+          BOOLEAN_FALSE_VALUES = %w(no n false f 0)
+
+          def as_boolean(value)
+            return nil if null_string?(value)
+
+            case value
+            when nil
+              nil
+            when String, Integer
+              return nil if value.is_a?(String) and value.empty?
+              value = value.to_s unless value.is_a?(String)
+              return true if BOOLEAN_TRUE_VALUES.include?(value.downcase)
+              return false if BOOLEAN_FALSE_VALUES.include?(value.downcase)
+              raise TypecastError.new "`embulk-input-google_spreadsheets`: cannot typecast '#{value}' to a boolean value." \
+                  " A boolean value must be one of #{BOOLEAN_TRUE_VALUES + BOOLEAN_FALSE_VALUES}: \"#{value}\""
+            when TrueClass
+              true
+            when FalseClass
+              false
+              # when Integer then ... => merge with String case
+            when Float
+              raise TypecastError.new "`embulk-input-google_spreadsheets`: cannot typecast Float to a boolean value: \"#{value}\""
+            when Array
+              raise TypecastError.new "`embulk-input-google_spreadsheets`: cannot typecast Array to a boolean value: \"#{value}\""
+            when Hash
+              raise TypecastError.new "`embulk-input-google_spreadsheets`: cannot typecast Hash to a boolean value: \"#{value}\""
+            when Time
+              raise TypecastError.new "`embulk-input-google_spreadsheets`: cannot typecast Time to String: \"#{value}\""
+            else
+              raise TypecastError.new "`embulk-input-google_spreadsheets`: cannot typecast #{value.class} to a boolean value: \"#{value}\""
+            end
+          end
+
+          def as_timestamp(value, timestamp_format = nil, timezone = nil)
+            return nil if null_string?(value)
+
+            case value
+            when nil
+              nil
+            when String, Integer, Float
+              return nil if value.is_a?(String) and value.empty?
+              value = value.to_s unless value.is_a?(String)
+              begin
+                if timestamp_format and TimestampFormatUtil.timezone_format?(timestamp_format)
+                  Time.strptime(value, timestamp_format)
+                elsif timestamp_format and timezone
+                  TimeWithZone.strptime_with_zone(value, timestamp_format, timezone)
+                elsif timezone
+                  TimeWithZone.parse_with_zone(value, timezone)
+                elsif timestamp_format
+                  Time.strptime(value, timestamp_format)
+                else
+                  Time.parse(value)
+                end
+              rescue ArgumentError => e
+                raise TypecastError.new "`embulk-input-google_spreadsheets`: cannot typecast #{value.class} to Time: \"#{value}\" because of '#{e}'"
+              end
+            when TrueClass
+              raise TypecastError.new "`embulk-input-google_spreadsheets`: cannot typecast true to Time: \"#{value}\""
+            when FalseClass
+              raise TypecastError.new "`embulk-input-google_spreadsheets`: cannot typecast false to Time: \"#{value}\""
+              # when Integer
+              # ... => merge with String case
+              # when Float
+              # ... => merge with String case
+            when Array
+              raise TypecastError.new "`embulk-input-google_spreadsheets`: cannot typecast Array to Time: \"#{value}\""
+            when Hash
+              raise TypecastError.new "`embulk-input-google_spreadsheets`: cannot typecast Hash to Time: \"#{value}\""
+            when Time
+              value
+            else
+              raise TypecastError.new "`embulk-input-google_spreadsheets`: cannot typecast #{value.class} to Time: \"#{value}\""
+            end
+          end
+
+          def as_json(value)
+            return nil if null_string?(value)
+
+            # NOTE: This method must do `JSON.parse` if `value` is String. ref. https://github.com/embulk/embulk/issues/499
+            case value
+            when nil, Array, Hash
+              value
+            when String
+              begin
+                JSON.parse(value)
+              rescue JSON::ParserError => e
+                raise TypecastError.new "`embulk-input-google_spreadsheets`: cannot typecast #{value.class} to JSON: \"#{value}\" because of '#{e}'"
+              end
+              # when TrueClass  then ... => merge with Time case
+              # when FalseClass then ... => merge with Time case
+              # when Integer    then ... => merge with Time case
+              # when Float      then ... => merge with Time case
+              # when Array      then ... => merge with nil case
+              # when Hash       then ... => merge with nil case
+            when TrueClass, FalseClass, Integer, Float, Array, Hash, Time
+              # TODO: support Time class. Now call Exception to avoid format/timezone trouble.
+              raise TypecastError.new "`embulk-input-google_spreadsheets`: cannot typecast Time to JSON: \"#{value}\""
+
+            else
+              raise TypecastError.new "`embulk-input-google_spreadsheets`: cannot typecast #{value.class} to JSON: \"#{value}\""
+            end
+          end
+
+        end
+      end
+    end
+  end
+end

data/lib/embulk/input/google_spreadsheets/typecast/timestamp_format_util.rb

@@ -0,0 +1,29 @@
+module Embulk
+  module Input
+    class GoogleSpreadsheets < InputPlugin
+      module Typecast
+        module TimestampFormatUtil
+
+          def self.timezone_format?(format)
+            @cache ||= {}
+            return @cache[format.dup] if @cache.has_key?(format)
+            @cache[format] = tz_regexp === format
+          end
+
+          private
+
+          def self.tz_regexp
+            # Time zone:
+            #   %z - Time zone as hour and minute offset from UTC (e.g. +0900)
+            #          %:z - hour and minute offset from UTC with a colon (e.g. +09:00)
+            #          %::z - hour, minute and second offset from UTC (e.g. +09:00:00)
+            #   %Z - Abbreviated time zone name or similar information.  (OS dependent)
+            #
+            # ref. https://docs.ruby-lang.org/en/2.3.0/Time.html#method-i-strftime
+            @tz_regexp ||= %r{(?:\A|[^%]|(?:%%)+)%(?::?:?z|Z)}
+          end
+        end
+      end
+    end
+  end
+end

data/lib/embulk/input/google_spreadsheets/typecast_factory.rb

@@ -0,0 +1,34 @@
+require_relative 'error'
+
+module Embulk
+  module Input
+    class GoogleSpreadsheets < InputPlugin
+      module TypecastFactory
+        def self.create(type, task)
+          raise GoogleSpreadsheets::ConfigError.new("`embulk-input-google_spreadsheets`: unknown typecast '#{type}'") if type.nil? or !type.is_a?(String)
+
+          type = type.downcase
+          path = build_typecast_class_path(type)
+          raise GoogleSpreadsheets::ConfigError.new("`embulk-input-google_spreadsheets`: Typecast class path does not exist '#{path}'") unless File.exist?(path)
+
+          require path
+          typecast_class(type).new(task)
+        end
+
+        private
+
+        def self.typecast_class(type)
+          Object.const_get("Embulk::Input::GoogleSpreadsheets::Typecast::#{camelize(type)}Typecast")
+        end
+
+        def self.build_typecast_class_path(type)
+          File.expand_path("typecast/#{type}_typecast.rb", __dir__)
+        end
+
+        def self.camelize(snake)
+          snake.split('_').map{|w| w[0] = w[0].upcase; w}.join
+        end
+      end
+    end
+  end
+end

data/test/assert_embulk_nothing_raised.rb

@@ -0,0 +1,11 @@
+module AssertEmbulkNothingRaised
+  # NOTE: assert_raise(TypecastError) do ... end can't detect correctly
+  def assert_embulk_nothing_raised(&block)
+    begin
+      yield
+      assert true
+    rescue => e
+      assert_equal(nil, e)
+    end
+  end
+end

data/test/assert_embulk_raise.rb

@@ -0,0 +1,11 @@
+module AssertEmbulkRaise
+  # NOTE: assert_raise(TypecastError) do ... end can't detect correctly
+  def assert_embulk_raise(klass, &block)
+    begin
+      yield
+      assert false
+    rescue => e
+      assert e.is_a?(klass), "expect: #{klass.name}, actual: #{e.class.name}, message: #{e}"
+    end
+  end
+end

data/test/dummy.key

@@ -0,0 +1,27 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIEpgIBAAKCAQEApmXpbr2WNslz+T881vt8uljjBtcJR8eu0UVcGfLjdZ+sn86z
+HMJgzKUlV5Y8GFT2ktgcznAM/Tn83XsKbOLKfdutdnRHQK/bG4Qj7QxI1TiEbJ8y
+c8EWdO8GvFQXkKSlL26U/t8tPgnFrYKt5C/XNvVL3vPEWS9MtJk903llWhRueXJ8
+iOFMKWq50fIr28gzDnzXqwErhzdDuxI2FWHcjXI20eWMg/uHG8m0OfHDZhcsOlri
++SXX+TkCwEeyUak2WqPCA+HzlNJRGyphgQIMixI8ZAnZ6vmz4coD5yXNRGOzAhq1
+86s/ajOAFnSOVN98SutvkrgY9UlywPv+5UVxnwIDAQABAoIBAQCeVt3xlhDJ45sr
+M8TWIguHW6v1R0y6dEZjNyg0PQ+7Tw06KLwCAumP8xp8pSZcVQUpqOKIgtGpabO4
+oivCf3EGoLMIxA9Fe9H49h4lpQntc9F3d2nQAo43CXkUXsTpy/KwFv9LJN3taekk
+efS07d1Sm8rtxnMyFrNcK3JMuPj75GmdSHPGnBth/EUMoNzE4FiTUJaDg/DrXHg/
+zxIW7H+vVKEnlhExp8vX4VXgXJJ1J4woXAD1h2RneV2zHu/OsXI5prBbSUdR2jSw
+obZqF2XFpBvZOOuYWQ53kJB1vqLWODb14Bp7okfpqA9QSM52tzTJjXYGpgd0lhGq
+vHfulyCJAoGBAN0Uw9slMitG3TlabKdCbzKHhw8izuyFpfPU+NU+PeKZo4yN5iRG
+jr5S+VlmavqzNyhgG6bh1f2e95Tygcu7CKMTDG6m4s71iZ+yr+yN/bX07DwSERor
+q3NtYxLtvAdEUkpAxLTDZXzYDYWXKOkslV8hNZJXZiQX98KdkEkljnbdAoGBAMCu
+FVZVVm186yQLE3myfHemyiGoyLNivEXDL0/L5P/FFTlS7CwMq3+ZBdPWuPKi8pjb
+K4NUxDOAWMgTJzKcQRCNLez470l9GGp+tIvlo7gtt8BVkIL0jIZ92xV7nGskDV1z
+HnqdkJ/v8Y2/CIZ9q7Q+iLcM+Xf/BNuccIwInXyrAoGBAL/r797FmpjhKtlIO3yz
++o5WAxRLtnCz0Q4su1t7bgPSNp2k7b5eFjr5EwfeFk7O17pQPJESCk/i+bEvGSiA
+lI3k0m0YqVra4v9lAr4Lwc188rtvP7OFZ5CLzG85PHref8WAFoOKAQdxE8P+JoDN
+uFpeHQCOqNf6ipZDbGdjmBptAoGBAL3H+b5CadGH0/d5bWJV2mDkCH3kyGJgIv83
+xBb9xn3ieggvU+4DQLx6VZIVEa/zra2/WnmN+X6csVXH/7jKzQ9nb0/U1aP1siHa
+0SxmML8NUMsBvEgwZWVm+26KhUPGi996rw+ouLW/2ZV7DwOhLolnerOJ9QIrTnac
+/QIAzlXnAoGBAMq/PA22fsGGyJwtKoxJiFwm6myb9q4dx7CsRUW8FivXXa8IBV6K
+8TvzytIGA9SjIRHQR9wDOlxyO7JE5ndEwBIsPesARvcRdrukN7UWorT5y33KRe8m
+bd54ICAxK+uNVWBv2xaM6V9hFdzeYZ8McZvlht4AqDmkHu/nsTYMeLci
+-----END RSA PRIVATE KEY-----

data/test/helper.rb

@@ -0,0 +1,21 @@
+#!/usr/bin/env ruby
+
+require 'test/unit'
+require 'test/unit/rr'
+
+ENV['TZ'] = 'JST-9'
+
+require 'embulk'
+Embulk.setup
+Embulk.logger = Embulk::Logger.new('/dev/null')
+
+APP_ROOT = File.expand_path('../', __dir__)
+EXAMPLE_ROOT = File.join(APP_ROOT, 'example')
+TEST_ROOT = File.join(APP_ROOT, 'test')
+JSON_KEYFILE_SERVICE_ACCOUNT = File.join(EXAMPLE_ROOT, 'service_account_credentials.json')
+JSON_KEYFILE_AUTHORIZED_USER = File.join(EXAMPLE_ROOT, 'authorized_user_credentials.json')
+TEST_SPREADSHEETS_URL = 'https://docs.google.com/spreadsheets/d/1Cxz-LudQuhRAGZL8mBoHs6mRnpjODpyF4Rwc5UYoV1E/edit#gid=0'
+TEST_SPREADSHEETS_ID = '1Cxz-LudQuhRAGZL8mBoHs6mRnpjODpyF4Rwc5UYoV1E'
+TEST_WORKSHEET_TITLE = 'sheet1'
+TEST_WORKSHEET_TITLE_MULTI_BYTE = '日本語（japanese）'
+DUMMY_RSA_KEY = File.expand_path('dummy.key', __dir__) # openssl genrsa 2048 > dummy.key

data/test/test_auth.rb

@@ -0,0 +1,82 @@
+require_relative 'helper'
+require 'embulk/input/google_spreadsheets'
+require_relative 'assert_embulk_raise'
+require_relative 'assert_embulk_nothing_raised'
+
+module Embulk
+  class Input::GoogleSpreadsheets < InputPlugin
+
+    class TestAuth < Test::Unit::TestCase
+
+      include AssertEmbulkRaise
+      include AssertEmbulkNothingRaised
+
+      sub_test_case 'authorized_user' do
+        unless File.exist?(JSON_KEYFILE_AUTHORIZED_USER)
+          puts "#{JSON_KEYFILE_AUTHORIZED_USER} is not found. Skip correct cases authorized_user"
+        else
+          test 'correct json_keyfile' do
+            task = {
+              'auth_method'  => 'authorized_user',
+              'json_keyfile' => File.read(JSON_KEYFILE_AUTHORIZED_USER)
+            }
+            assert_embulk_nothing_raised do
+              Auth.new(task).authenticate.fetch_access_token!
+            end
+          end
+        end
+
+        test 'incorrect json_keyfile' do
+          task = {
+            'auth_method'  => 'authorized_user',
+            'json_keyfile' => '{"client_id":"hoge", "client_secret":"fuga","refresh_token":"hogo"}'
+          }
+          assert_embulk_raise Signet::AuthorizationError do
+            Auth.new(task).authenticate.fetch_access_token!
+          end
+        end
+      end
+
+      sub_test_case 'service_account' do
+        unless File.exist?(JSON_KEYFILE_SERVICE_ACCOUNT)
+          puts "#{JSON_KEYFILE_SERVICE_ACCOUNT} is not found. Skip correct cases service_account"
+        else
+          test 'correct json_keyfile' do
+            task = {
+              'auth_method'  => 'service_account',
+              'json_keyfile' => File.read(JSON_KEYFILE_SERVICE_ACCOUNT)
+            }
+            assert_embulk_nothing_raised do
+              Auth.new(task).authenticate.fetch_access_token!
+            end
+          end
+        end
+
+        test 'incorrect json_keyfile' do
+          dummy_rsa_key = File.read(DUMMY_RSA_KEY).gsub("\n", '\n')
+          task = {
+            'auth_method'  => 'service_account',
+            'json_keyfile' => '{"client_id":"hoge", "client_secret":"fuga","client_email":"hogo","private_key":"' + dummy_rsa_key + '"}'
+          }
+          assert_embulk_raise Signet::AuthorizationError do
+            Auth.new(task).authenticate.fetch_access_token!
+          end
+        end
+      end
+
+      # TODO: compute_engine cases
+      # TODO: application_default cases
+
+      sub_test_case 'ConfigError' do
+        test 'invalid auth_method' do
+          task = {
+            'auth_methqd' => 'hoge'
+          }
+          assert_embulk_raise ConfigError do
+            Auth.new(task).authenticate
+          end
+        end
+      end
+    end
+  end
+end