bricolage-spreadsheet 1.0.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 465aebc471ab4d6adfac54aedc78c2766204d202c49009c29c848e1ef391519e
+  data.tar.gz: b8d4b1e847bd67d8f865ed9b5fd42ba05f0d34a08a30787166d1bee09a2b0320
+SHA512:
+  metadata.gz: 35d5f63b44638ffc54ff4279a60e95e50337137fb635c2fe5c517db0ccf0fd6ae731427fe5997575610b8e3213eaedb9f513d73c9e51fc64c81eebc46af6e7ee
+  data.tar.gz: add032b531a4d56067e1cf937746777a1be317084b0c763afb2079551fc7407cbd3d4fdb698ec38ae96772eb9a09f624d7ec4dde3004966c31813ed4f77eb494

data/README.md

@@ -0,0 +1,50 @@
+# bricolage-spreadsheet
+
+Google Spreadsheet-related job classes for Bricolage batch job framework.
+
+## Home Page
+
+https://github.com/bricolages/bricolage-spreadsheet
+
+## Usage
+
+Add following line in your Gemfile:
+```
+gem 'bricolage-spreadsheet'
+```
+
+Job Options
+
+```
+% bundle exec bricolage spreadsheet-import -h
+Usage: bricolage spreadsheet-import [job_class_options]
+        --src-ds=NAME                [optional] Main data source. [default: spreadsheet]
+        --sheet-id=ID                Spreadsheet ID
+        --range=RANGE_EXPR           Spreadsheet Range
+        --format=VALUE               [optional] Data file format. (csv, json)
+        --value-render-option=VALUE  [optional] For values with format on sheets (FORMATTED_VALUE, UNFORMATTED_VALUE, FORMULA)
+        --s3-ds=NAME                 [optional] Main data source. [default: s3]
+        --s3-file=PATH               Target file name.
+        --dest-ds=NAME               [optional] Main data source. [default: psql]
+        --dest-table=[SCHEMA.]TABLE  [optional] Target table name.
+        --options=OPTIONS            [optional] Loader options.
+        --table-def=PATH             Create table file.
+        --no-backup                  [optional] Drop dest table with suffix "_old".
+        --analyze                    [optional] ANALYZE table after SQL is executed.
+        --grant=KEY:VALUE            [optional] GRANT table after SQL is executed. (required keys: privilege, to)
+        --gzip                       [optional] Compress Temporary files.
+    -v, --variable=NAME=VALUE        Set variable.
+        --help                       Shows this message and quit.
+        --version                    Shows program version and quit.
+```
+
+## License
+
+MIT license.
+See LICENSES file for details.
+
+## Credit
+
+Author: Shimpei Kodama
+
+This software is written in working time in Cookpad, Inc.

data/jobclass/spreadsheet-import.rb

@@ -0,0 +1,72 @@
+require 'bricolage/psqldatasource'
+
+module Bricolage
+  JobClass.define('spreadsheet-import') {
+    parameters {|params|
+      # S3Export
+      params.add DataSourceParam.new('spreadsheet', 'src-ds')
+      params.add StringParam.new('sheet-id', 'ID', 'Google Spreadsheet ID')
+      params.add StringParam.new('range', 'RANGE_EXPR', 'Google Spreadsheet Range Expression')
+      params.add EnumParam.new('format', %w(csv json), 'Intermediate data file format.', default: 'json')
+      params.add EnumParam.new('value-render-option', %w(FORMATTED_VALUE UNFORMATTED_VALUE FORMULA), 'For values with format on sheets', default: 'FORMATTED_VALUE')
+      params.add DataSourceParam.new('s3', 's3-ds')
+      params.add DestFileParam.new('s3-file')
+
+      # Load
+      params.add DataSourceParam.new('psql', 'dest-ds')
+      params.add DestTableParam.new
+      params.add KeyValuePairsParam.new('options', 'OPTIONS', 'Loader options.',
+          optional: true, default: PSQLLoadOptions.new,
+          value_handler: lambda {|value, ctx, vars| PSQLLoadOptions.parse(value) })
+      params.add SQLFileParam.new('table-def', 'PATH', 'Create table file.')
+      params.add OptionalBoolParam.new('no-backup', 'Drop dest table with suffix "_old".', default: false)
+
+      # Misc
+      params.add OptionalBoolParam.new('analyze', 'ANALYZE table after SQL is executed.', default: true)
+      params.add KeyValuePairsParam.new('grant', 'KEY:VALUE', 'GRANT table after SQL is executed. (required keys: privilege, to)')
+      params.add OptionalBoolParam.new('gzip', 'Compress Temporary files.')
+    }
+
+    script {|params, script|
+      # S3Export
+      script.task(params['src-ds']) {|task|
+        task.s3export params['sheet-id'],
+                      params['range'],
+                      params['format'],
+                      params['value-render-option'],
+                      params['s3-ds'],
+                      params['s3-file'],
+                      params['gzip']
+      }
+
+      # Load
+      script.task(params['dest-ds']) {|task|
+        prev_table = '${dest_table}_old'
+        work_table = '${dest_table}_wk'
+
+        task.transaction {
+          # CREATE
+          task.drop_force prev_table
+          task.drop_force work_table
+          task.exec params['table-def'].replace(/\$\{?dest_table\}?\b/, work_table)
+
+          # COPY
+          options = params['gzip'] ? params['options'].merge('gzip' => params['gzip']) : params['options']
+          task.load params['s3-ds'], params['s3-file'], work_table,
+              params['format'], nil, options
+
+          # GRANT, ANALYZE
+          task.grant_if params['grant'], work_table
+          task.analyze_if params['analyze'], work_table
+
+          # RENAME
+          task.create_dummy_table '${dest_table}'
+          task.rename_table params['dest-table'].to_s, "#{params['dest-table'].name}_old"
+          task.rename_table work_table, params['dest-table'].name
+        }
+        # No Backup
+        task.drop_force prev_table if params['no-backup']
+      }
+    }
+  } #spreadsheet-import job class
+end

data/lib/bricolage-spreadsheet.rb

@@ -0,0 +1,5 @@
+require 'bricolage/jobclass'
+require 'pathname'
+
+jobclass_path = Pathname(__dir__).realpath.parent.cleanpath + 'jobclass'
+Bricolage::JobClass.add_load_path jobclass_path

data/lib/bricolage/spreadsheetdatasource.rb

@@ -0,0 +1,181 @@
+require 'bricolage/datasource'
+require 'bricolage/jobresult'
+require 'google/apis/sheets_v4'
+require 'googleauth'
+require 'csv'
+require 'json'
+require 'uri'
+require 'tempfile'
+require 'zlib'
+require 'pathname'
+
+module Bricolage
+
+  class SpreadsheetDataSource < DataSource
+    declare_type 'spreadsheet'
+
+    SCOPE_BASE = "Google::Apis::SheetsV4::"
+    DEFAULT_SCOPE = 'AUTH_SPREADSHEETS_READONLY'
+    DEFAULT_APPLICATION_NAME = 'Bricolage'
+
+    def initialize(credentials: nil, scope: nil, application_name: nil)
+      @credentials = credentials
+      @scope = "#{SCOPE_BASE}#{(scope || DEFAULT_SCOPE)}"
+      @application_name = application_name || DEFAULT_APPLICATION_NAME
+    end
+
+    attr_reader :scope, :application_name
+
+    def new_task
+      SpreadsheetTask.new(self)
+    end
+
+    def rows(sheet_id, range, **get_options)
+      return enum_for(:rows, sheet_id, range, **get_options) unless block_given?
+      response = service.get_spreadsheet_values(sheet_id, range, **get_options)
+      response.values.each do |row|
+        next if row.all?(&:empty?) # skip empty row
+        yield row
+      end
+    end
+
+    def formatted_rows(sheet_id, range, format = 'csv', **get_options)
+      return enum_for(:formatted_rows, sheet_id, range, format, **get_options) unless block_given?
+      row_formatter = RowFormatterFactory.new_formatter(format)
+      fields = []
+      rows(sheet_id, range, **get_options).each_with_index do |row, idx|
+        if idx == 0
+          fields = row
+          next if row_formatter.skip_header?
+        end
+        yield row_formatter.format(row, fields)
+      end
+    end
+
+    def credential
+      if @credentials
+        case
+        when @credentials.is_a?(Hash) then StringIO.new(@credentials.to_json)
+        when Pathname.new(@credentials).exist? then File.open(@credentials)
+        else raise ParameterError, "credentials must be a JSON or PATH. credentials=#{@credentials}"
+        end
+      elsif ENV['GOOGLE_APPLICATION_CREDENTIALS']
+        File.open(ENV['GOOGLE_APPLICATION_CREDENTIALS'])
+      else
+        raise ParameterError, "credentials or GOOGLE_APPLICATION_CREDENTIALS is required."
+      end
+    end
+
+    class SpreadsheetTask < DataSourceTask
+
+      def s3export(sheet_id, range, format, value_render_option, dest_ds, dest_file, gzip)
+        add S3ExportAction.new(sheet_id, range, format, value_render_option, dest_ds, dest_file, gzip)
+      end
+
+      class S3ExportAction < Action
+        def initialize(sheet_id, range, format, value_render_option, dest_ds, dest_file, gzip)
+          @sheet_id = sheet_id
+          @range = range
+          @format = format
+          @value_render_option = value_render_option
+          @dest_ds = dest_ds
+          @dest_file = dest_file
+          @gzip = gzip
+        end
+
+        attr_reader :sheet_id, :range, :dest_ds, :dest_file, :gzip
+
+        def url_encoded_range
+          @url_encoded_range ||= URI.encode(range)
+        end
+
+        def value_render_option
+          @value_render_option&.upcase
+        end
+
+        def format
+          @format.downcase
+        end
+
+        def source
+          <<~SOURCE
+            "GET https://sheets.googleapis.com/v4/spreadsheets/#{sheet_id}/values/#{url_encoded_range}"
+            "PUT s3://#{dest_ds.bucket_name}/#{dest_ds.prefix}/#{dest_file}"
+          SOURCE
+        end
+
+        def run
+          ds.logger.info source
+          rows = ds.formatted_rows(sheet_id, range, format, value_render_option: value_render_option)
+          Tempfile.open do |f|
+            f = Zlib::GzipWriter.wrap(f) if gzip
+            f.write rows.to_a.join("\n")
+            f.close # flush
+            dest_ds.object(dest_file).upload_file(f.path)
+          end
+          nil
+        end
+
+      end #S3ExportAction
+    end #SpreadsheetTask
+
+    private
+
+    def service
+      return @service if @service
+      @service = Google::Apis::SheetsV4::SheetsService.new
+      @service.client_options.application_name = application_name
+      @service.authorization = authorizer
+      @service
+    end
+
+    def authorizer
+      return @authorizer if @authorizer
+      @authorizer = Google::Auth::ServiceAccountCredentials.make_creds(
+        json_key_io: credential,
+        scope: scope,
+      )
+      # token lifetime is 3600 sec
+      # May need to implement refresh logic for long running app???
+      @authorizer.fetch_access_token!
+      @authorizer
+    end
+
+    class RowFormatterFactory
+      def self.new_formatter(format)
+        const_get(format.upcase! + "Formatter").new
+      end
+
+      class CSVFormatter
+        def format(values, fields)
+          # remove  '\n' with row_sep: nil
+          escaped_values = values.map {|v| v.gsub(/\n/,'\\n') }
+          CSV.generate_line(escaped_values, row_sep: nil, quote_char: '"', force_quotes: true)
+        end
+
+        def skip_header?
+          false
+        end
+      end
+
+      class JSONFormatter
+        def format(values, fields)
+          # https://stackoverflow.com/questions/1509915/converting-camel-case-to-underscore-case-in-ruby
+          normalized_fields = fields.map {|f|
+            f.gsub(/::/, '/').
+            gsub(/([A-Z]+)([A-Z][a-z])/,'\1_\2').
+            gsub(/([a-z\d])([A-Z])/,'\1_\2').
+            tr("-", "_").
+            tr(" ", "_").
+            downcase
+          }
+          normalized_fields.zip(values).to_h.to_json
+        end
+
+        def skip_header?
+          true
+        end
+      end
+    end # RowFormatterFactory
+  end
+end

metadata

@@ -0,0 +1,74 @@
+--- !ruby/object:Gem::Specification
+name: bricolage-spreadsheet
+version: !ruby/object:Gem::Version
+  version: 1.0.0
+platform: ruby
+authors:
+- Shimpei Kodama
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2021-04-19 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bricolage
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 5.26.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 5.26.0
+- !ruby/object:Gem::Dependency
+  name: google-apis-sheets_v4
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.4.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.4.0
+description: 
+email: shimpeko@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- README.md
+- jobclass/spreadsheet-import.rb
+- lib/bricolage-spreadsheet.rb
+- lib/bricolage/spreadsheetdatasource.rb
+homepage: https://github.com/bricolages/bricolage-spreadsheet
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - "~>"
+    - !ruby/object:Gem::Version
+      version: 2.7.0
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.1.4
+signing_key: 
+specification_version: 4
+summary: Google Spreadsheet-related job classes for Bricolage batch framework
+test_files: []