embulk-input-google_spreadsheets 1.0.0

data/example/config_service_account_large_data.yml

@@ -0,0 +1,19 @@
+in:
+  type: google_spreadsheets
+  auth_method: service_account
+  json_keyfile: example/service_account_credentials.json
+  spreadsheets_url: https://docs.google.com/spreadsheets/d/1Cxz-LudQuhRAGZL8mBoHs6mRnpjODpyF4Rwc5UYoV1E/edit#gid=0
+  worksheet_title: Large Data
+  start_row: 2
+  default_timezone: Asia/Tokyo
+  null_string: '\N'
+  default_typecast: strict
+  columns:
+    - {name: _c1, type: boolean}
+    - {name: _c2, type: string}
+    - {name: _c3, type: long}
+    - {name: _c4, type: double}
+    - {name: _c5, type: timestamp, format: '%Y-%m-%d %H:%M:%S.%N'}
+    - {name: _c6, type: timestamp, format: '%Y-%m-%d'}
+out:
+  type: stdout

data/example/config_service_account_no_data.yml

@@ -0,0 +1,18 @@
+in:
+  type: google_spreadsheets
+  auth_method: service_account
+  json_keyfile: example/service_account_credentials.json
+  spreadsheets_url: https://docs.google.com/spreadsheets/d/1Cxz-LudQuhRAGZL8mBoHs6mRnpjODpyF4Rwc5UYoV1E/edit#gid=0
+  worksheet_title: No Data
+  default_timezone: Asia/Tokyo
+  null_string: '\N'
+  default_typecast: strict
+  columns:
+    - {name: _c1, type: boolean}
+    - {name: _c2, type: string}
+    - {name: _c3, type: long}
+    - {name: _c4, type: double}
+    - {name: _c5, type: timestamp, format: '%Y-%m-%d %H:%M:%S.%N'}
+    - {name: _c6, type: timestamp, format: '%Y-%m-%d'}
+out:
+  type: stdout

data/example/setup_authorized_user_credentials.rb

@@ -0,0 +1,34 @@
+require 'googleauth'
+require 'google/apis/sheets_v4'
+require 'highline/import'
+require 'json'
+
+puts 'Before setup, open this page https://developers.google.com/identity/protocols/OAuth2'
+puts 'then get OAuth 2.0 credentials such as a client ID and client secret according to the above page.'
+puts
+
+credentials = Google::Auth::UserRefreshCredentials.new(
+  client_id: ask('Enter client_id: '),
+  client_secret: ask('Enter client_secret: '),
+  scope: Google::Apis::SheetsV4::AUTH_SPREADSHEETS_READONLY,
+  redirect_uri: 'urn:ietf:wg:oauth:2.0:oob'
+)
+
+credentials.code = ask(
+  "1. Open this page '#{credentials.authorization_uri.to_s}'.\n" \
+  '2. Enter the authorization code shown in the page: '
+) {|q| q.echo = false}
+
+credentials.fetch_access_token!
+
+data = {
+  client_id: credentials.client_id,
+  client_secret: credentials.client_secret,
+  refresh_token: credentials.refresh_token,
+}.to_json
+file = File.expand_path('authorized_user_credentials.json', __dir__)
+File.open(file, 'w') do |f|
+  f.write(data)
+end
+
+puts "Success. See '#{file}'."

data/lib/embulk/input/google_spreadsheets.rb

@@ -0,0 +1,182 @@
+require_relative 'google_spreadsheets/error'
+require_relative 'google_spreadsheets/record_typecaster'
+require_relative 'google_spreadsheets/auth'
+require_relative 'google_spreadsheets/spreadsheets_client'
+require_relative 'google_spreadsheets/pager'
+
+module Embulk
+  module Input
+
+    class GoogleSpreadsheets < InputPlugin
+      Plugin.register_input('google_spreadsheets', self)
+
+      # support config by file path or content which supported by org.embulk.spi.unit.LocalFile
+      # json_keyfile:
+      #   content: |
+      class LocalFile
+        # return JSON string
+        def self.load(v)
+          if v.is_a?(String)
+            File.read(v)
+          elsif v.is_a?(Hash)
+            v['content']
+          end
+        end
+      end
+
+      class CustomColumns
+        # NOTE: if raised, rescue and re-raise as Embulk::ConfigError
+        def self.load(v)
+          raise "`embulk-input-google_spreadsheets`: Invalid value '#{v}' for :array_of_hash" unless v.is_a?(Array)
+          v.each do |c|
+            raise "`embulk-input-google_spreadsheets`: Invalid value '#{v}' for :array_of_hash" unless c.is_a?(Hash)
+          end
+
+          complete_default(v.dup)
+        end
+
+        def self.complete_default(columns)
+          columns.map do |c|
+            c = c.dup
+            if c['type'] == 'timestamp'
+              c['format'] = c['format'] || default_format
+              c['timezone'] = c['timezone'] || default_timezone
+            end
+            c['typecast']  = c['typecast'] || default_typecast
+            c
+          end
+        end
+
+        def self.default_format
+          # ref. https://github.com/embulk/embulk/blob/936c5d5a20af3086f7d1e5779a89035105bb975b/embulk-core/src/main/java/org/embulk/spi/type/TimestampType.java#L10
+          # `Time.strptime` does not support `%6N`, so use `%N` instead.
+          @default_format ||= '%Y-%m-%d %H:%M:%S.%N %z'
+        end
+
+        def self.default_format=(format)
+          @default_format = format
+        end
+
+        def self.default_timezone
+          @default_timezone ||= 'UTC'
+        end
+
+        def self.default_timezone=(timezone)
+          @default_timezone = timezone
+        end
+
+        def self.default_typecast
+          @default_typecast ||= 'strict'
+        end
+
+        def self.default_typecast=(typecast)
+          @default_typecast = typecast
+        end
+      end
+
+      def self.logger
+        ::Embulk.logger
+      end
+
+      def logger
+        self.class.logger
+      end
+
+      def self.configure(config)
+        task = {}
+        # auth_method:
+        #   - service_account
+        #   - authorized_user
+        #   - compute_engine
+        #   - application_default
+        task['auth_method'] = config.param('auth_method', :string, default: 'authorized_user')
+        # json_keyfile: Fullpath of json key
+        #   if `auth_method` is `authorized_user`, this plugin supposes the format
+        #   is the below.
+        #   {
+        #     "client_id":"xxxxxxxxxxx.apps.googleusercontent.com",
+        #     "client_secret":"xxxxxxxxxxx",
+        #     "refresh_token":"xxxxxxxxxxx"
+        #   }
+        #
+        #   if `auth_method` is `compute_engine` or `application_default`, this
+        #   option is not required.
+        task['json_keyfile']           = config.param('json_keyfile',          LocalFile, default: nil)
+        task['spreadsheets_url']       = config.param('spreadsheets_url',       :string)
+        task['worksheet_title']        = config.param('worksheet_title',        :string)
+        task['start_column']           = config.param('start_column',           :integer, default: 1)
+        task['start_row']              = config.param('start_row',              :integer, default: 1)
+        task['end_row']                = config.param('end_row',                :integer, default: -1)
+        task['max_fetch_rows']         = config.param('max_fetch_rows',         :integer, default: 10000)
+        task['null_string']            = config.param('null_string',            :string,  default: '')
+        task['stop_on_invalid_record'] = config.param('stop_on_invalid_record', :bool,    default: true)
+        # columns: this option supposes an array of hash has the below structure.
+        #   - name
+        #   - type
+        #   - format
+        #   - timezone
+        #   - typecast: default: strict
+        CustomColumns.default_format   = task['default_timestamp_format'] = config.param('default_timestamp_format', :string, default: CustomColumns.default_format)
+        CustomColumns.default_timezone = task['default_timezone']         = config.param('default_timezone',         :string, default: CustomColumns.default_timezone)
+        CustomColumns.default_typecast = task['default_typecast']         = config.param('default_typecast',         :string, default: CustomColumns.default_typecast)
+        task['columns'] = config.param('columns', CustomColumns)
+
+        task['end_column'] = task['start_column'] + task['columns'].length - 1
+
+        logger.debug { "`embulk-input-google_spreadsheets`: configured task '#{task.reject{|k, v| k == 'json_keyfile'}.to_json}'"}
+        task
+      end
+
+      def self.configure_columns(task)
+        task['columns'].map.with_index do |c, i|
+          Column.new(i, c['name'], c['type'].to_sym, c['format'])
+        end
+      end
+
+      def self.transaction(config, &control)
+        task = configure(config)
+        columns = configure_columns(task)
+        resume(task, columns, 1, &control)
+      end
+
+      def self.resume(task, columns, count, &control)
+        task_reports = yield(task, columns, count)
+
+        next_config_diff = {}
+        return next_config_diff
+      end
+
+      attr_reader :typecaster, :client
+
+      def init
+        @typecaster = RecordTypecaster.new(task)
+        @client = SpreadsheetsClient.new(task, auth: Auth.new(task), pager: Pager.new(task))
+      end
+
+      def stop_on_invalid_record?
+        task['stop_on_invalid_record']
+      end
+
+      def run
+        client.worksheet_each_record do |record|
+          begin
+            record = typecaster.transform_by_columns(record)
+            page_builder.add(record)
+          rescue => e
+            if stop_on_invalid_record?
+              raise e if e.is_a?(ConfigError) or e.is_a?(DataError)
+              raise DataError.new(e)
+            end
+            logger.warn{ "`embulk-input-google_spreadsheets`: Error '#{e}' occurred. Skip '#{record}'" }
+          end
+        end
+
+        page_builder.finish
+
+        task_report = {}
+        return task_report
+      end
+    end
+  end
+end
+

data/lib/embulk/input/google_spreadsheets/auth.rb

@@ -0,0 +1,63 @@
+require 'googleauth'
+require 'google/apis/sheets_v4'
+
+module Embulk
+  module Input
+    class GoogleSpreadsheets < InputPlugin
+      class Auth
+
+        attr_reader :auth_method
+
+        def initialize(task)
+          @auth_method = task['auth_method']
+          @json_key = task['json_keyfile']
+        end
+
+        def authenticate
+          case auth_method
+          when 'authorized_user'
+            key = StringIO.new(credentials.to_json)
+            return Google::Auth::UserRefreshCredentials.make_creds(json_key_io: key, scope: scope)
+          when 'compute_engine'
+            return Google::Auth::GCECredentials.new
+          when 'service_account'
+            key = StringIO.new(credentials.to_json)
+            return Google::Auth::ServiceAccountCredentials.make_creds(json_key_io: key, scope: scope)
+          when 'application_default'
+            return Google::Auth.get_application_default([scope])
+          else
+            raise ConfigError.new("Unknown auth method: #{auth_method}")
+          end
+        end
+
+        def scope
+          Google::Apis::SheetsV4::AUTH_SPREADSHEETS_READONLY
+        end
+
+        private
+
+        def credentials
+          JSON.parse(@json_key || File.read(credentials_file))
+        end
+
+        def credentials_file
+          @credentials_file ||= File.expand_path(
+            # ref. https://developers.google.com/identity/protocols/application-default-credentials
+            (File.exist?(global_application_default_credentials_file) ?
+              global_application_default_credentials_file : application_default_credentials_file)
+          )
+        end
+
+        def application_default_credentials_file
+          @application_default_credentials_file ||=
+            File.expand_path('~/.config/gcloud/application_default_credentials.json')
+        end
+
+        def global_application_default_credentials_file
+          @global_application_default_credentials_file ||=
+            '/etc/google/auth/application_default_credentials.json'
+        end
+      end
+    end
+  end
+end

data/lib/embulk/input/google_spreadsheets/error.rb

@@ -0,0 +1,36 @@
+module Embulk
+  module Input
+
+    class GoogleSpreadsheets < InputPlugin
+
+      module Traceable
+        def initialize(e, more_msg = nil)
+          message = e.is_a?(String) ? '' : "(#{e.class}) "
+          message << "#{e}#{more_msg}\n"
+          message << "\tat #{e.backtrace.join("\n\tat ")}\n" if e.respond_to?(:backtrace)
+
+          while e.respond_to?(:cause) and e.cause
+            # Java Exception cannot follow the JRuby causes.
+            message << "Caused by (#{e.cause.class}) #{e.cause}\n"
+            message << "\tat #{e.cause.backtrace.join("\n\tat ")}\n" if e.cause.respond_to?(:backtrace)
+            e = e.cause
+          end
+
+          super(message)
+        end
+      end
+
+      class ConfigError < ::Embulk::ConfigError
+        include Traceable
+      end
+
+      class DataError < ::Embulk::DataError
+        include Traceable
+      end
+
+      class TypecastError < DataError
+      end
+
+    end
+  end
+end

data/lib/embulk/input/google_spreadsheets/pager.rb

@@ -0,0 +1,107 @@
+require_relative 'pager_util'
+
+module Embulk
+  module Input
+    class GoogleSpreadsheets < InputPlugin
+      class Pager
+        attr_reader :start_row, :start_column, :end_row, :end_column, :max_fetch_rows
+
+        def initialize(task)
+          @start_row = task['start_row']
+          @start_column = task['start_column']
+          @end_row = task['end_row']
+          @end_column = task['end_column']
+          @max_fetch_rows = task['max_fetch_rows']
+
+          validate!
+        end
+
+        def logger
+          GoogleSpreadsheets.logger
+        end
+
+        def each_record(client, &block)
+          max_row_num = max_accessible_row_num(client)
+
+          total_fetched_rows = 0
+          last_fetched_row_num = start_row - 1
+          while true do
+            start_row_num = last_fetched_row_num + 1
+            end_row_num = last_fetched_row_num + max_fetch_rows
+            if end_row_num >= max_row_num
+              end_row_num = max_row_num
+            end
+
+            range = range(start_row_num, end_row_num)
+            page = client.worksheet_values(range)
+            unless page # no values
+              logger.warn { '`embulk-input-google_spreadsheets`: no data is found.' } if total_fetched_rows <= 0
+              break
+            end
+
+            num_fetched_rows = 0
+            page.each do |record|
+              break false if no_limit? and empty_record?(record)
+              num_fetched_rows += 1
+              yield(record)
+            end
+            total_fetched_rows = total_fetched_rows + num_fetched_rows
+            logger.info { "`embulk-input-google_spreadsheets`: fetched #{num_fetched_rows} rows in #{range} (tatal: #{total_fetched_rows} rows)" }
+            break if num_fetched_rows < max_fetch_rows
+
+            last_fetched_row_num = end_row_num
+            break if last_fetched_row_num >= max_row_num
+          end
+        end
+
+        private
+
+        def validate!
+          if (has_limit? && start_row > end_row) || start_column > end_column
+            raise ConfigError.new("`embulk-input-google_spreadsheets`: Area does not exist. Please check start & end for row and column. start_row: '#{start_row}', end_row: '#{end_row}', start_col: '#{start_column}', end_col: '#{end_column}'")
+          end
+          if max_fetch_rows <= 0
+            raise ConfigError.new('`embulk-input-google_spreadsheets`: `max_fetch_rows` must be positive integer.')
+          end
+        end
+
+        def max_accessible_row_num(client)
+          sheets_max = client.worksheet_max_row_num
+          if end_row > sheets_max
+            raise ConfigError.new("`embulk-input-google_spreadsheets`: end_row `#{end_row}` is larger than spreadsheets max row `#{sheets_max}`")
+          end
+
+          return sheets_max if no_limit?
+
+          end_row
+        end
+
+        def empty_record?(record)
+          return true unless record
+          return true if record.empty?
+          record.all?{|v| v.nil? or v.empty?}
+        end
+
+        def no_limit?
+          end_row <= 0
+        end
+
+        def has_limit?
+          !no_limit?
+        end
+
+        def start_column_name
+          @start_column_name ||= PagerUtil.num2col(start_column)
+        end
+
+        def end_column_name
+          @end_column_name ||= PagerUtil.num2col(end_column)
+        end
+
+        def range(start_row_num, end_row_num)
+          "#{start_column_name}#{start_row_num}:#{end_column_name}#{end_row_num}"
+        end
+      end
+    end
+  end
+end