embulk-input-google_spreadsheets 1.0.0

data/lib/embulk/input/google_spreadsheets/pager_util.rb

@@ -0,0 +1,28 @@
+module Embulk
+  module Input
+    class GoogleSpreadsheets < InputPlugin
+      module PagerUtil
+
+        def self.num2col(num, base = default_base, offset = default_offset)
+          [].tap do |r|
+            while num > 0
+              num -= 1
+              r.unshift((num % base + offset).chr)
+              num /= base
+            end
+          end.join
+        end
+
+        private
+
+        def self.default_offset
+          @default_offset ||= 'A'.ord
+        end
+
+        def self.default_base
+          @default_base ||= 26 # number of alphabet
+        end
+      end
+    end
+  end
+end

data/lib/embulk/input/google_spreadsheets/record_typecaster.rb

@@ -0,0 +1,73 @@
+require_relative 'typecast_factory'
+
+
+module Embulk
+  module Input
+    class GoogleSpreadsheets < InputPlugin
+      class RecordTypecaster
+
+        attr_reader :column_names, :column_details
+
+        def initialize(task)
+          @column_names = task['columns'].map{|c| c['name']}
+          @column_details = configure_column_details(task)
+        end
+
+        def configure_column_details(task)
+          _column_details = task['columns'].dup.each_with_index.inject({}) do |details, column_with_index|
+            c, i = *column_with_index
+            details.tap do |ds|
+              ds[c['name']] = {}.tap do |d|
+                d['index'] = i
+                d['name'] = c['name']
+                d['type'] = c['type'].to_sym
+                d['format'] = c['format']
+                d['timezone'] = c['timezone']
+                d['typecast'] = TypecastFactory.create(c['typecast'], task)
+              end
+            end
+          end
+
+          logger.debug { "`embulk-input-google_spreadsheets`: configured column details '#{_column_details.to_json}'"}
+          _column_details
+        end
+
+        def logger
+          GoogleSpreadsheets.logger
+        end
+
+        def transform_by_columns(record)
+          column_names.map do |n|
+            d = column_details[n]
+            typecast = d['typecast']
+            value = record[d['index']]
+            type = d['type']
+
+            begin
+              case type
+              when :string
+                typecast.as_string(value)
+              when :long
+                typecast.as_long(value)
+              when :double
+                typecast.as_double(value)
+              when :boolean
+                typecast.as_boolean(value)
+              when :timestamp
+                typecast.as_timestamp(value, d['format'], d['timezone'])
+              when :json
+                typecast.as_json(value)
+              else
+                raise ConfigError.new("`google_spreadsheets`: Unsupported type `#{type}`")
+              end
+            rescue => e
+              # for adding column information
+              raise TypecastError.new(e, ", column: #{n}, column_detail: #{d.to_json}")
+            end
+          end
+        end
+
+      end
+    end
+  end
+end

data/lib/embulk/input/google_spreadsheets/spreadsheets_client.rb

@@ -0,0 +1,75 @@
+require 'google/apis/sheets_v4'
+require_relative 'spreadsheets_url_util'
+
+module Embulk
+  module Input
+    class GoogleSpreadsheets < InputPlugin
+
+      class SpreadsheetsClient
+
+        attr_accessor :spreadsheets_url, :worksheet_title, :auth, :pager
+
+        def initialize(task, auth:, pager:)
+          @spreadsheets_url = task['spreadsheets_url']
+          @worksheet_title = task['worksheet_title']
+          @auth = auth
+          @pager = pager
+        end
+
+        def logger
+          GoogleSpreadsheets.logger
+        end
+
+        def application_name
+          @application_name ||= 'embulk-input-google_spreadsheets'
+        end
+
+        def spreadsheets_id
+          SpreadsheetsUrlUtil.capture_id(spreadsheets_url)
+        end
+
+        def spreadsheets
+          service.get_spreadsheet(spreadsheets_id, ranges: worksheet_title, include_grid_data: false)
+        end
+
+        def worksheet
+          spreadsheets.sheets.first
+        end
+
+        def worksheet_properties
+          worksheet.properties
+        end
+
+        def worksheet_grid_properties
+          worksheet_properties.grid_properties
+        end
+
+        def worksheet_max_row_num
+          worksheet_grid_properties.row_count
+        end
+
+        def worksheet_max_column_num
+          worksheet_grid_properties.column_count
+        end
+
+        def worksheet_values(range)
+          range = "#{worksheet_title}!#{range}"
+          logger.info { "`embulk-input-google_spreadsheets`: load data from spreadsheet: '#{spreadsheets_url}', range: '#{range}'" }
+          service.get_spreadsheet_values(spreadsheets_id, range).values
+        end
+
+        def worksheet_each_record(&block)
+          pager.each_record(self, &block)
+        end
+
+        def service
+          @service ||= Google::Apis::SheetsV4::SheetsService.new.tap do |s|
+            s.client_options.application_name = application_name
+            s.authorization = auth.authenticate
+          end
+        end
+
+      end
+    end
+  end
+end

data/lib/embulk/input/google_spreadsheets/spreadsheets_url_util.rb

@@ -0,0 +1,23 @@
+module Embulk
+  module Input
+    class GoogleSpreadsheets < InputPlugin
+      module SpreadsheetsUrlUtil
+
+        def self.capture_id(url)
+          scanned = url.scan(capture_id_regex).first
+          return unless scanned
+          scanned.first
+        end
+
+        def self.base_url
+          @base_url ||= 'https://docs.google.com/spreadsheets/d/'
+        end
+
+        def self.capture_id_regex
+          @capture_id_regex ||= %r{#{base_url}([^/]+)/.*}
+        end
+      end
+    end
+  end
+end
+

data/lib/embulk/input/google_spreadsheets/typecast/base.rb

@@ -0,0 +1,62 @@
+module Embulk
+  module Input
+    class GoogleSpreadsheets < InputPlugin
+      module Typecast
+        class Base
+
+          attr_reader :null_string
+
+          def initialize(task)
+            @null_string = task['null_string']
+          end
+
+          def logger
+            GoogleSpreadsheets.logger
+          end
+
+          def to_json(*args) # for logging
+            spec = {JSON.create_id => self.class.name}
+            spec = instance_variables.inject(spec) do |spec, v|
+              spec.tap do |s|
+                s[v] = instance_variable_get(v)
+              end
+            end
+            spec.to_json(*args)
+          end
+
+          def as_string(value)
+            raise NotImplementedError, '`embulk-input-google_spreadsheets`: override this.'
+          end
+
+          def as_long(value)
+            raise NotImplementedError, '`embulk-input-google_spreadsheets`: override this.'
+          end
+
+          def as_double(value)
+            raise NotImplementedError, '`embulk-input-google_spreadsheets`: override this.'
+          end
+
+          def as_boolean(value)
+            raise NotImplementedError, '`embulk-input-google_spreadsheets`: override this.'
+          end
+
+          def as_timestamp(value, timestamp_format, timezone)
+            raise NotImplementedError, '`embulk-input-google_spreadsheets`: override this.'
+          end
+
+          def as_json(value)
+            raise NotImplementedError, '`embulk-input-google_spreadsheets`: override this.'
+          end
+
+          protected
+
+          def null_string?(value)
+            return false unless value.is_a?(String)
+            return true if value == null_string
+            return false
+          end
+        end
+      end
+    end
+  end
+end

data/lib/embulk/input/google_spreadsheets/typecast/loose_typecast.rb

@@ -0,0 +1,84 @@
+require_relative 'strict_typecast'
+
+module Embulk
+  module Input
+    class GoogleSpreadsheets < InputPlugin
+      module Typecast
+        class LooseTypecast < StrictTypecast
+          def as_string(value)
+            begin
+              super
+            rescue => e
+              if e.is_a?(TypecastError)
+                logger.trace{"`embulk-input-google_spreadsheets`: Fallback to nil, because of '#{e}'"}
+                return nil
+              end
+              raise e
+            end
+          end
+
+          def as_long(value)
+            begin
+              super
+            rescue => e
+              if e.is_a?(TypecastError)
+                logger.trace{"`embulk-input-google_spreadsheets`: Fallback to nil, because of '#{e}'"}
+                return nil
+              end
+              raise e
+            end
+          end
+
+          def as_double(value)
+            begin
+              super
+            rescue => e
+              if e.is_a?(TypecastError)
+                logger.trace{"`embulk-input-google_spreadsheets`: Fallback to nil, because of '#{e}'"}
+                return nil
+              end
+              raise e
+            end
+          end
+
+          def as_boolean(value)
+            begin
+              super
+            rescue => e
+              if e.is_a?(TypecastError)
+                logger.trace{"`embulk-input-google_spreadsheets`: Fallback to nil, because of '#{e}'"}
+                return nil
+              end
+              raise e
+            end
+          end
+
+          def as_timestamp(value, timestamp_format = nil, timezone = nil)
+            begin
+              super
+            rescue => e
+              if e.is_a?(TypecastError)
+                logger.trace{"`embulk-input-google_spreadsheets`: Fallback to nil, because of '#{e}'"}
+                return nil
+              end
+              raise e
+            end
+          end
+
+          def as_json(value)
+            begin
+              super
+            rescue => e
+              if e.is_a?(TypecastError)
+                logger.trace{"`embulk-input-google_spreadsheets`: Fallback to nil, because of '#{e}'"}
+                return nil
+              end
+              raise e
+            end
+          end
+
+        end
+      end
+    end
+  end
+end

data/lib/embulk/input/google_spreadsheets/typecast/minimal_typecast.rb

@@ -0,0 +1,109 @@
+require 'time_with_zone'
+require_relative 'base'
+require_relative 'timestamp_format_util'
+
+module Embulk
+  module Input
+    class GoogleSpreadsheets < InputPlugin
+      module Typecast
+        class MinimalTypecast < Base
+
+          def as_string(value)
+            return nil if value.nil?
+            return nil if null_string?(value)
+            value.to_s
+          rescue NoMethodError => e
+            raise TypecastError.new "`embulk-input-google_spreadsheets`: cannot typecast #{value.class} to String: \"#{value}\" because of '#{e}'"
+          end
+
+          def as_long(value)
+            return nil if value.nil?
+            return nil if null_string?(value)
+            value.to_i
+          rescue NoMethodError => e
+            raise TypecastError.new "`embulk-input-google_spreadsheets`: cannot typecast #{value.class} to Long: \"#{value}\" because of '#{e}'"
+          end
+
+          def as_double(value)
+            return nil if value.nil?
+            return nil if null_string?(value)
+            value.to_f
+          rescue NoMethodError => e
+            raise TypecastError.new "`embulk-input-google_spreadsheets`: cannot typecast #{value.class} to Double: \"#{value}\" because of '#{e}'"
+          end
+
+          def as_boolean(value)
+            return nil if value.nil?
+            return nil if null_string?(value)
+
+            case value
+            when TrueClass, FalseClass
+              value
+            when String
+              value = value.downcase
+              case value
+              when 'true'
+                true
+              when 'false'
+                false
+              else
+                raise TypecastError.new "`embulk-input-google_spreadsheets`: cannot typecast '#{value}' to a boolean value."
+              end
+            else
+              raise TypecastError.new "`embulk-input-google_spreadsheets`: cannot typecast #{value.class} to a boolean value: \"#{value}\""
+            end
+          end
+
+          def as_timestamp(value, timestamp_format = nil, timezone = nil)
+            return nil if value.nil?
+            return nil if null_string?(value)
+
+            if timestamp_format and TimestampFormatUtil.timezone_format?(timestamp_format)
+              Time.strptime(value, timestamp_format)
+            elsif timestamp_format and timezone
+              TimeWithZone.strptime_with_zone(value, timestamp_format, timezone)
+            elsif timezone
+              TimeWithZone.parse_with_zone(value, timezone)
+            elsif timestamp_format
+              Time.strptime(value, timestamp_format)
+            else
+              Time.parse(value)
+            end
+          rescue ArgumentError, TypeError, NoMethodError => e
+            raise TypecastError.new "`embulk-input-google_spreadsheets`: cannot typecast #{value.class} to Time: \"#{value}\" because of '#{e}'"
+          end
+
+          def as_json(value)
+            return nil if value.nil?
+            return nil if null_string?(value)
+
+            # cf. https://github.com/embulk/embulk/blob/191ffd50e555565be77f810db15a21ba66cb7bf6/lib/embulk/page_builder.rb#L20
+            # cf. https://github.com/embulk/embulk/blob/191ffd50e555565be77f810db15a21ba66cb7bf6/embulk-core/src/main/java/org/embulk/spi/util/DynamicPageBuilder.java#L97
+            # cf. https://github.com/embulk/embulk/blob/191ffd50e555565be77f810db15a21ba66cb7bf6/embulk-core/src/main/java/org/embulk/spi/util/DynamicColumnSetterFactory.java#L66
+            # cf. https://github.com/embulk/embulk/blob/997c7beb89d42122f7cb6fe844f8ca79a3cb666c/embulk-core/src/main/java/org/embulk/spi/util/dynamic/JsonColumnSetter.java#L50
+            # cf. https://github.com/embulk/embulk/blob/191ffd50e555565be77f810db15a21ba66cb7bf6/embulk-core/src/main/java/org/embulk/spi/util/dynamic/AbstractDynamicColumnSetter.java#L47
+            # cf. https://github.com/embulk/embulk/blob/191ffd50e555565be77f810db15a21ba66cb7bf6/embulk-core/src/main/java/org/embulk/spi/json/RubyValueApi.java#L57
+            # NOTE: As long as reading the above code, any object can be set as Json
+            #       (that must be primitive type or must have `to_msgpack` method.)
+            case value
+            when TrueClass, FalseClass, Integer, Float, Array, Hash
+              value
+            when String
+              begin
+                JSON.parse(value)
+              rescue JSON::ParserError => e
+                raise TypecastError.new "`embulk-input-google_spreadsheets`: cannot typecast #{value.class} to JSON: \"#{value}\" because of '#{e}'"
+              end
+            when Time
+              # TODO: support Time class. Now call Exception to avoid format/timezone trouble.
+              raise TypecastError.new "`embulk-input-google_spreadsheets`: cannot typecast Time to JSON: \"#{value}\""
+            else
+              raise TypecastError.new "`embulk-input-google_spreadsheets`: cannot typecast #{value.class} to JSON: \"#{value}\""
+            end
+          end
+
+        end
+      end
+    end
+  end
+end