embulk-input-google_spreadsheets 1.0.0 → 1.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: '0298679088554e589d5fcf35d28052321996397b'
-  data.tar.gz: 437d4de2a144529e3a12c734bf2742c475632c4b
+  metadata.gz: 9f7dd9c3e53a70bcc237da3b46ea67f2ac70c207
+  data.tar.gz: 8cf701fd4cfe2dbef37ea6246517382b6b01f4ec
 SHA512:
-  metadata.gz: f92a35388a40ac64fe15dde0eab169db9471884b4e27742ee5bb98ed94f994a528f6ad57c0d7400095953ac8864ba7907c600297b5a2d3f2df27096bbbab530e
-  data.tar.gz: 0c391fa5d167d32471d3881a77cc64012fe42a6aafa76f8a13250ef734a87d32d8aab24320db72b6b423ee2056deffc606ce2ad5f8aa9300fd591619b19f64ab
+  metadata.gz: 5d2ea9909f5c8f62362715916d7a2fe7096a4f2e51738b52a4573737dfa372bcb7d71dff41b15286618cfea45eee798fb08301178cc83709ca1a421d464e3d4a
+  data.tar.gz: 961a7934ee7b3acf93be1c9ac17265789378a54ff10fe155a07cacf0069b3ca8be9ebb35ea899b17ae60afc2323d823cef12a7af6ef6e9dd33b3cc7bb2f2034d

data/CHANGELOG.md

@@ -1,3 +1,8 @@
+1.1.0 (2017-04-18)
+==================
+- New Feature: Support `value_render_option` option.
+  - https://github.com/medjed/embulk-input-google_spreadsheets/pull/9
+
 1.0.0 (2017-03-27)
 ==================
 

data/README.md

@@ -21,6 +21,7 @@ Embulk input plugin to load records from Google Spreadsheets.
 | end_row            | integer     | optional    | `-1`            | `-1` means loading records until an empty record appears. |
 | max_fetch_rows     | integer     | optional    | `10000`        |  Load data from a worksheet for each numerical value specified by this option. |
 | null_string        | string      | optional    | `''`            |  Replace this value to `NULL` |
+| value_render_option| string      | optional    | `'FORMATTED_VALUE'`| `'FORMATTED_VALUE'`, `'UNFORMATTED_VALUE'`, `'FORMULA'` are available. See [the `value_render_option` document](https://developers.google.com/sheets/api/reference/rest/v4/ValueRenderOption).|
 | stop_on_invalid_record | boolean | optional    | `true`          |  |
 | default_timestamp_format | string | optional | `'%Y-%m-%d %H:%M:%S.%N %z'` | |
 | default_timezone | string | optional | `'UTC'` | |

data/embulk-input-google_spreadsheets.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |spec|
   spec.name          = 'embulk-input-google_spreadsheets'
-  spec.version       = '1.0.0'
+  spec.version       = '1.1.0'
   spec.authors       = %w(Civitaspo yang-xu)
   spec.summary       = 'Google Spreadsheets input plugin for Embulk'
   spec.description   = 'Load records from Google Spreadsheets.'
@@ -19,6 +19,6 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency 'test-unit'
   spec.add_development_dependency 'test-unit-rr'
 
-  spec.add_dependency 'google-api-client', '>= 0.9.15'
+  spec.add_dependency 'google-api-client', '>= 0.11'
   spec.add_dependency 'time_with_zone'
 end

data/example/config_authorized_user_unformatted_value.yml

@@ -0,0 +1,15 @@
+in:
+  type: google_spreadsheets
+  auth_method: authorized_user
+  json_keyfile: example/authorized_user_credentials.json
+  spreadsheets_url: https://docs.google.com/spreadsheets/d/1Cxz-LudQuhRAGZL8mBoHs6mRnpjODpyF4Rwc5UYoV1E/edit#gid=0
+  worksheet_title: UNFORMATTED VALUE
+  start_row: 2
+  default_timezone: Asia/Tokyo
+  null_string: '\N'
+  default_typecast: strict
+  value_render_option: UNFORMATTED_VALUE
+  columns:
+    - {name: _c1, type: long}
+out:
+  type: stdout

data/example/config_service_account_unformatted_value.yml

@@ -0,0 +1,15 @@
+in:
+  type: google_spreadsheets
+  auth_method: service_account
+  json_keyfile: example/service_account_credentials.json
+  spreadsheets_url: https://docs.google.com/spreadsheets/d/1Cxz-LudQuhRAGZL8mBoHs6mRnpjODpyF4Rwc5UYoV1E/edit#gid=0
+  worksheet_title: UNFORMATTED VALUE
+  start_row: 2
+  default_timezone: Asia/Tokyo
+  null_string: '\N'
+  default_typecast: strict
+  value_render_option: UNFORMATTED_VALUE
+  columns:
+    - {name: _c1, type: long}
+out:
+  type: stdout

data/lib/embulk/input/google_spreadsheets.rb

@@ -108,6 +108,9 @@ module Embulk
         task['start_row']              = config.param('start_row',              :integer, default: 1)
         task['end_row']                = config.param('end_row',                :integer, default: -1)
         task['max_fetch_rows']         = config.param('max_fetch_rows',         :integer, default: 10000)
+        # FORMATTED_VALUE, UNFORMATTED_VALUE, FORMULA are available.
+        # ref. https://developers.google.com/sheets/api/reference/rest/v4/ValueRenderOption
+        task['value_render_option']    = config.param('value_render_option',    :string,  default: 'FORMATTED_VALUE')
         task['null_string']            = config.param('null_string',            :string,  default: '')
         task['stop_on_invalid_record'] = config.param('stop_on_invalid_record', :bool,    default: true)
         # columns: this option supposes an array of hash has the below structure.

data/lib/embulk/input/google_spreadsheets/pager.rb

@@ -79,7 +79,11 @@ module Embulk
         def empty_record?(record)
           return true unless record
           return true if record.empty?
-          record.all?{|v| v.nil? or v.empty?}
+          record.all? do |v|
+            next true if v.nil?
+            next true if v.is_a?(String) and v.empty?
+            false
+          end
         end
 
         def no_limit?

data/lib/embulk/input/google_spreadsheets/spreadsheets_client.rb

@@ -7,11 +7,12 @@ module Embulk
 
       class SpreadsheetsClient
 
-        attr_accessor :spreadsheets_url, :worksheet_title, :auth, :pager
+        attr_accessor :spreadsheets_url, :worksheet_title, :value_render_option, :auth, :pager
 
         def initialize(task, auth:, pager:)
           @spreadsheets_url = task['spreadsheets_url']
           @worksheet_title = task['worksheet_title']
+          @value_render_option = task['value_render_option']
           @auth = auth
           @pager = pager
         end
@@ -54,8 +55,11 @@ module Embulk
 
         def worksheet_values(range)
           range = "#{worksheet_title}!#{range}"
-          logger.info { "`embulk-input-google_spreadsheets`: load data from spreadsheet: '#{spreadsheets_url}', range: '#{range}'" }
-          service.get_spreadsheet_values(spreadsheets_id, range).values
+          logger.info {
+            "`embulk-input-google_spreadsheets`: load data from spreadsheet: '#{spreadsheets_url}'," \
+            " range: '#{range}', value_render_option: '#{value_render_option}'"
+          }
+          service.get_spreadsheet_values(spreadsheets_id, range, value_render_option: value_render_option).values
         end
 
         def worksheet_each_record(&block)

data/test/test_configure.rb

@@ -54,6 +54,7 @@ module Embulk
           assert_equal(-1, task['end_row'])
           assert_equal(10000, task['max_fetch_rows'])
           assert_equal('', task['null_string'])
+          assert_equal('FORMATTED_VALUE', task['value_render_option'])
           assert_equal(true, task['stop_on_invalid_record'])
           assert_equal('%Y-%m-%d %H:%M:%S.%N %z', task['default_timestamp_format'])
           assert_equal('UTC', task['default_timezone'])

data/test/test_run_examples.rb

@@ -46,7 +46,7 @@ module Embulk
           puts "'#{JSON_KEYFILE_AUTHORIZED_USER}' is not found. Skip case authorized_user"
         else
           test 'authorized_user' do
-            omit 'Skip until closing https://github.com/medjed/embulk-input-google_spreadsheets/issues/6'
+            omit 'Skip until closing https://github.com/medjed/embulk-input-google_spreadsheets/issues/7'
             assert_true embulk_run(File.join(EXAMPLE_ROOT, 'config_authorized_user_emoji_worksheet.yml'))
           end
         end
@@ -55,7 +55,7 @@ module Embulk
           puts "'#{JSON_KEYFILE_SERVICE_ACCOUNT}' is not found. Skip case service_account"
         else
           test 'service_account' do
-            omit 'Skip until closing https://github.com/medjed/embulk-input-google_spreadsheets/issues/6' 
+            omit 'Skip until closing https://github.com/medjed/embulk-input-google_spreadsheets/issues/7'
             assert_true embulk_run(File.join(EXAMPLE_ROOT, 'config_service_account_emoji_worksheet.yml'))
           end
         end
@@ -120,6 +120,26 @@ module Embulk
           end
         end
       end
+
+      sub_test_case 'run unformatted value' do
+        unless File.exist?(JSON_KEYFILE_AUTHORIZED_USER)
+          puts "'#{JSON_KEYFILE_AUTHORIZED_USER}' is not found. Skip case authorized_user"
+        else
+          test 'authorized_user' do
+
+            assert_true embulk_run(File.join(EXAMPLE_ROOT, 'config_authorized_user_unformatted_value.yml'))
+          end
+        end
+
+        unless File.exist?(JSON_KEYFILE_SERVICE_ACCOUNT)
+          puts "'#{JSON_KEYFILE_SERVICE_ACCOUNT}' is not found. Skip case service_account"
+        else
+          test 'service_account' do
+
+            assert_true embulk_run(File.join(EXAMPLE_ROOT, 'config_service_account_unformatted_value.yml'))
+          end
+        end
+      end
     end
   end
 end

data/test/test_spreadsheets_client.rb

@@ -75,8 +75,6 @@ else
         end
 
         test 'multi byte worksheet title' do
-          omit 'Skip until closing this issue https://github.com/medjed/embulk-input-google_spreadsheets/issues/6'
-
           assert_embulk_nothing_raised do
             client('worksheet_title' => TEST_WORKSHEET_TITLE_MULTI_BYTE).worksheet_values('A1:C1')
           end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: embulk-input-google_spreadsheets
 version: !ruby/object:Gem::Version
-  version: 1.0.0
+  version: 1.1.0
 platform: ruby
 authors:
 - Civitaspo
@@ -9,7 +9,7 @@ authors:
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2017-03-29 00:00:00.000000000 Z
+date: 2017-04-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -100,7 +100,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.9.15
+        version: '0.11'
   name: google-api-client
   prerelease: false
   type: :runtime
@@ -108,7 +108,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.9.15
+        version: '0.11'
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
@@ -145,11 +145,13 @@ files:
 - example/config_authorized_user_empty_rows_appears_at_the_same_as_max_fetch_rows.yml
 - example/config_authorized_user_large_data.yml
 - example/config_authorized_user_no_data.yml
+- example/config_authorized_user_unformatted_value.yml
 - example/config_service_account.yml
 - example/config_service_account_emoji_worksheet.yml
 - example/config_service_account_empty_rows_appears_at_the_same_as_max_fetch_rows.yml
 - example/config_service_account_large_data.yml
 - example/config_service_account_no_data.yml
+- example/config_service_account_unformatted_value.yml
 - example/setup_authorized_user_credentials.rb
 - lib/embulk/input/google_spreadsheets.rb
 - lib/embulk/input/google_spreadsheets/auth.rb