embulk-output-bigquery 0.7.0 → 0.7.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 36a45d63bf6d79411d67ef26d736e3a55035220b92923f208c8e08c7af5aeeae
-  data.tar.gz: 3be9cdc0cfd8c05cc7d6f427ba6aa9a4091220c8eb5b4cef8786cde8d86a86bd
+  metadata.gz: b3f1f3663f59942ba67caf8d3aa2785df824aaef5cc09de0bdcbadadd3642d81
+  data.tar.gz: 389b3111be5db9924f67fd2ce3d5d2c0afbe46d12c44c7df6ee5c95e04269c79
 SHA512:
-  metadata.gz: a112e890334fa11d95ebfa40719654f7c6a78ea3fcda19edc141060f68516b748353ca76c2f1e35d1b2ea6ba9aec852f6279f91d0d57879387786465e8651d03
-  data.tar.gz: 776ad5a340e420874b5ff9daa2860797fe9a656ce2f8fc3585ab4aff8d671f2725628803cacccfe54c10a68acefe956f73aa3b717a243aebd1566bf68ab7068f
+  metadata.gz: 5136502bb9003aa85f85c9c5d3b5c826f218476a7604f7d201160bf4931936f8f02b00fb77ea286b9ae8fc69b007b122a1f7f0707cec4a7b7d030d1d1771ce86
+  data.tar.gz: c16e57af1121e30e0bcfa187311d9f7652768a2bda79354202ce7bac92e9e5bf3bc33cdb2fbb95696f03653fa830f844f95bd01e200cb0b7d9732f2a3b989bad

data/.github/workflows/check.yml

@@ -9,34 +9,25 @@ jobs:
     strategy:
       matrix:
         jruby_version:
-        - 9.3.10.0
-        - 9.4.2.0
+        - 9.3.15.0
+        - 9.4.8.0
       fail-fast: false
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
     - name: Set up OpenJDK 8
-      uses: actions/setup-java@v3
+      uses: actions/setup-java@v4
       with:
         java-version: 8
         distribution: "temurin"
-    - uses: ruby/setup-ruby@v1
-      with:
-        ruby-version: 'jruby-${{ matrix.jruby_version }}'
-        bundler-cache: true
-    - name: show ruby version
-      run: ruby -v
-    - name: bundle install
-      run: bundle install
+    - name: download jruby
+      run: "curl -L -o jruby.jar https://repo1.maven.org/maven2/org/jruby/jruby-complete/${{ matrix.jruby_version }}/jruby-complete-${{ matrix.jruby_version }}.jar"
     #
-    # This step avoids the following error in the JRuby 9.4 test.
+    # For avoiding permission denied. install gems into `gems` directory
     #
-    # Gem::LoadError: You have already activated rake 13.0.6,
-    # but your Gemfile requires rake 13.1.0. Prepending
-    # `bundle exec` to your command may solve this.
-    #
-    - name: install rake 13.1.0
-      run: gem install rake -v 13.1.0
+    - name: bundle install
+      run: "env GEM_HOME=gems java -jar jruby.jar -S bundle install"
+
     - name: install embulk.jar
-      run: "curl -L -o embulk.jar https://github.com/embulk/embulk/releases/download/v0.10.49/embulk-0.10.49.jar"
+      run: "curl -L -o embulk.jar https://github.com/embulk/embulk/releases/download/v0.11.4/embulk-0.11.4.jar"
     - name: rake test
-      run: bundle exec env RUBYOPT="-r ./embulk.jar" rake test
+      run: 'env GEM_HOME=gems RUBYOPT="-r ./embulk.jar -r rubygems" java -jar jruby.jar -S bundle exec rake test'

data/CHANGELOG.md

@@ -1,3 +1,14 @@
+## 0.7.2 - 2024-07-21
+* [maintenance] Fix GitHub Actions #166
+* [maintenance] Fix gcs_client in order to load data using gcs_bucket parameter (Thanks to kashira202111) #164
+* [maintenance] Prevent creating unnecessary tables. (Thanks to kashira202111) #148
+
+## 0.7.1 - 2024-03-4
+* [enhancement] Support description of columns and tables (Thanks to @kyoshidajp and @fagai ) #142
+* [maintenance] Add missing GitHub Actions environment setting. #160
+* [maintenance] Replace google-api-client with specific Google APIs (Thanks to @Nozomuts) #161
+* [maintenance] Update GitHub Actions use checkout@v4 and setup-java@v4 #162
+
 ## 0.7.0 - 2024-02-1
 * [enhancement] Add support Embulk 0.11.x
 

data/Gemfile

@@ -1,7 +1,7 @@
 source 'https://rubygems.org/'
 
 gemspec
-gem 'embulk', '= 0.10.49'
+gem 'embulk', '= 0.11.4'
 gem 'embulk-parser-none'
 gem 'embulk-parser-jsonl'
 gem 'pry-nav'

data/README.md

@@ -60,6 +60,7 @@ OAuth flow for installed applications.
 |  gcs_bucket                          | string      | optional   | nil                      | See [GCS Bucket](#gcs-bucket) |
 |  auto_create_gcs_bucket              | boolean     | optional   | false                    | See [GCS Bucket](#gcs-bucket) |
 |  progress_log_interval               | float       | optional   | nil (Disabled)           | Progress log interval. The progress log is disabled by nil (default). NOTE: This option may be removed in a future because a filter plugin can achieve the same goal |
+|  description                         | string      | optional   | nil                      | description of table |
 
 Client or request options
 
@@ -329,6 +330,7 @@ Column options are used to aid guessing BigQuery schema, or to define conversion
     - json:      `STRING`,  `RECORD` (default: `STRING`)
   - **mode**: BigQuery mode such as `NULLABLE`, `REQUIRED`, and `REPEATED` (string, default: `NULLABLE`)
   - **fields**: Describes the nested schema fields if the type property is set to RECORD. Please note that this is **required** for `RECORD` column.
+  - **description**: description (string, default is `None`).
   - **timestamp_format**: timestamp format to convert into/from `timestamp` (string, default is `default_timestamp_format`)
   - **timezone**: timezone to convert into/from `timestamp`, `date` (string, default is `default_timezone`).
 - **default_timestamp_format**: default timestamp format for column_options (string, default is "%Y-%m-%d %H:%M:%S.%6N")

data/embulk-output-bigquery.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |spec|
   spec.name          = "embulk-output-bigquery"
-  spec.version       = "0.7.0"
+  spec.version       = "0.7.2"
   spec.authors       = ["Satoshi Akama", "Naotoshi Seo"]
   spec.summary       = "Google BigQuery output plugin for Embulk"
   spec.description   = "Embulk plugin that insert records to Google BigQuery."
@@ -15,7 +15,8 @@ Gem::Specification.new do |spec|
   spec.require_paths = ["lib"]
 
   # the latest version
-  spec.add_dependency 'google-api-client','= 0.53.0'
+  spec.add_dependency 'google-apis-storage_v1'
+  spec.add_dependency 'google-apis-bigquery_v2'
   spec.add_dependency 'time_with_zone'
   spec.add_dependency 'thwait'
   # activesupport require Ruby >= 2.7.0

data/lib/embulk/output/bigquery/bigquery_client.rb

@@ -420,6 +420,7 @@ module Embulk
               table_reference: {
                 table_id: table,
               },
+              description: @task['description'],
               schema: {
                 fields: fields,
               }
@@ -446,8 +447,8 @@ module Embulk
             end
 
             opts = {}
-            Embulk.logger.debug { "embulk-output-bigquery: insert_table(#{@project}, #{dataset}, #{@location_for_log}, #{body}, #{opts})" }
-            with_network_retry { client.insert_table(@project, dataset, body, **opts) }
+            Embulk.logger.debug { "embulk-output-bigquery: insert_table(#{@destination_project}, #{dataset}, #{@location_for_log}, #{body}, #{opts})" }
+            with_network_retry { client.insert_table(@destination_project, dataset, body, **opts) }
           rescue Google::Apis::ServerError, Google::Apis::ClientError, Google::Apis::AuthorizationError => e
             if e.status_code == 409 && /Already Exists:/ =~ e.message
               # ignore 'Already Exists' error
@@ -456,7 +457,7 @@ module Embulk
 
             response = {status_code: e.status_code, message: e.message, error_class: e.class}
             Embulk.logger.error {
-              "embulk-output-bigquery: insert_table(#{@project}, #{dataset}, #{@location_for_log}, #{body}, #{opts}), response:#{response}"
+              "embulk-output-bigquery: insert_table(#{@destination_project}, #{dataset}, #{@location_for_log}, #{body}, #{opts}), response:#{response}"
             }
             raise Error, "failed to create table #{@destination_project}:#{dataset}.#{table} in #{@location_for_log}, response:#{response}"
           end

data/lib/embulk/output/bigquery/gcs_client.rb

@@ -48,7 +48,7 @@ module Embulk
             opts = {}
 
             Embulk.logger.debug { "embulk-output-bigquery: insert_temporary_bucket(#{@project}, #{body}, #{opts})" }
-            with_network_retry { client.insert_bucket(@project, body, opts) }
+            with_network_retry { client.insert_bucket(@project, body, **opts) }
           rescue Google::Apis::ServerError, Google::Apis::ClientError, Google::Apis::AuthorizationError => e
             if e.status_code == 409 && /conflict:/ =~ e.message
               # ignore 'Already Exists' error
@@ -81,7 +81,7 @@ module Embulk
 
             Embulk.logger.debug { "embulk-output-bigquery: insert_object(#{bucket}, #{body}, #{opts})" }
             # memo: gcs is strongly consistent for insert (read-after-write). ref: https://cloud.google.com/storage/docs/consistency
-            with_network_retry { client.insert_object(bucket, body, opts) }
+            with_network_retry { client.insert_object(bucket, body, **opts) }
           rescue Google::Apis::ServerError, Google::Apis::ClientError, Google::Apis::AuthorizationError => e
             response = {status_code: e.status_code, message: e.message, error_class: e.class}
             Embulk.logger.error {
@@ -114,7 +114,7 @@ module Embulk
             opts = {}
 
             Embulk.logger.debug { "embulk-output-bigquery: delete_object(#{bucket}, #{object}, #{opts})" }
-            response = with_network_retry { client.delete_object(bucket, object, opts) }
+            response = with_network_retry { client.delete_object(bucket, object, **opts) }
           rescue Google::Apis::ServerError, Google::Apis::ClientError, Google::Apis::AuthorizationError => e
             if e.status_code == 404 # ignore 'notFound' error
               return nil

data/lib/embulk/output/bigquery/helper.rb

@@ -46,10 +46,11 @@ module Embulk
             embulk_type   = column[:type]
             column_option = column_options_map[column_name] || {}
             {}.tap do |field|
-              field[:name]   = column_name
-              field[:type]   = (column_option['type'] || bq_type_from_embulk_type(embulk_type)).upcase
-              field[:mode]   = column_option['mode'] if column_option['mode']
-              field[:fields] = deep_symbolize_keys(column_option['fields']) if column_option['fields']
+              field[:name]        = column_name
+              field[:type]        = (column_option['type'] || bq_type_from_embulk_type(embulk_type)).upcase
+              field[:mode]        = column_option['mode'] if column_option['mode']
+              field[:fields]      = deep_symbolize_keys(column_option['fields']) if column_option['fields']
+              field[:description] = column_option['description'] if column_option['description']
             end
           end
         end

data/lib/embulk/output/bigquery.rb

@@ -63,6 +63,8 @@ module Embulk
           'payload_column'                 => config.param('payload_column',                 :string,  :default => nil),
           'payload_column_index'           => config.param('payload_column_index',           :integer, :default => nil),
 
+          'description'                    => config.param('description',                    :string,  :default => nil),
+
           'open_timeout_sec'               => config.param('open_timeout_sec',               :integer, :default => nil),
           'timeout_sec'                    => config.param('timeout_sec',                    :integer, :default => nil), # google-api-ruby-client < v0.11.0
           'send_timeout_sec'               => config.param('send_timeout_sec',               :integer, :default => nil), # google-api-ruby-client >= v0.11.0

data/test/test_helper.rb

@@ -68,7 +68,7 @@ module Embulk
           ])
           task = {
             'column_options' => [
-              {'name' => 'boolean',   'type' => 'STRING', 'mode' => 'REQUIRED'},
+              {'name' => 'boolean',   'type' => 'STRING', 'mode' => 'REQUIRED', 'description' => 'hoge'},
               {'name' => 'long',      'type' => 'STRING'},
               {'name' => 'double',    'type' => 'STRING'},
               {'name' => 'string',    'type' => 'INTEGER'},
@@ -81,7 +81,7 @@ module Embulk
             ],
           }
           expected = [
-            {name: 'boolean',   type: 'STRING', mode: 'REQUIRED'},
+            {name: 'boolean', type: 'STRING', mode: 'REQUIRED', description: 'hoge'},
             {name: 'long',      type: 'STRING'},
             {name: 'double',    type: 'STRING'},
             {name: 'string',    type: 'INTEGER'},

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: embulk-output-bigquery
 version: !ruby/object:Gem::Version
-  version: 0.7.0
+  version: 0.7.2
 platform: ruby
 authors:
 - Satoshi Akama
@@ -9,22 +9,36 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2024-02-13 00:00:00.000000000 Z
+date: 2024-07-21 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  name: google-api-client
+  name: google-apis-storage_v1
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '='
+    - - ">="
       - !ruby/object:Gem::Version
-        version: 0.53.0
+        version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '='
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: google-apis-bigquery_v2
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
       - !ruby/object:Gem::Version
-        version: 0.53.0
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: time_with_zone
   requirement: !ruby/object:Gem::Requirement