embulk-output-bigquery 0.6.8 → 0.7.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: c09f158dadb5a3b59f6ffb41d0f9736274e33bcbfbf09898018c0a5055f75010
-  data.tar.gz: b62efb97686156f0fdf8597ed24322f7af3332682b07e5f13a2270f9cf5d8270
+  metadata.gz: 36a45d63bf6d79411d67ef26d736e3a55035220b92923f208c8e08c7af5aeeae
+  data.tar.gz: 3be9cdc0cfd8c05cc7d6f427ba6aa9a4091220c8eb5b4cef8786cde8d86a86bd
 SHA512:
-  metadata.gz: b108f379dae0cf5e8eedaff9be22de2234963b8fd57dfdc11b9458593cac27a99174e6d95e15919a54208c15cdcabe5f60c4ea7a0f959011db9f5e7c4e75042f
-  data.tar.gz: e3f276763e4efc8f83ef26d641540263f5127c5ad1c941a01b1634a1bdc5d87ac278e0e7172ea5249bfd010a1ddcbe5d97d442ccb35be7222ea87e84255d9f10
+  metadata.gz: a112e890334fa11d95ebfa40719654f7c6a78ea3fcda19edc141060f68516b748353ca76c2f1e35d1b2ea6ba9aec852f6279f91d0d57879387786465e8651d03
+  data.tar.gz: 776ad5a340e420874b5ff9daa2860797fe9a656ce2f8fc3585ab4aff8d671f2725628803cacccfe54c10a68acefe956f73aa3b717a243aebd1566bf68ab7068f

data/.github/workflows/check.yml

@@ -0,0 +1,42 @@
+name: Check
+on: [ pull_request, push ]
+jobs:
+  check:
+    runs-on: ubuntu-latest
+    # push: always run.
+    # pull_request: run only when the PR is submitted from a forked repository, not within this repository.
+    if: github.event_name == 'push' || github.event.pull_request.head.repo.full_name != github.repository
+    strategy:
+      matrix:
+        jruby_version:
+        - 9.3.10.0
+        - 9.4.2.0
+      fail-fast: false
+    steps:
+    - uses: actions/checkout@v3
+    - name: Set up OpenJDK 8
+      uses: actions/setup-java@v3
+      with:
+        java-version: 8
+        distribution: "temurin"
+    - uses: ruby/setup-ruby@v1
+      with:
+        ruby-version: 'jruby-${{ matrix.jruby_version }}'
+        bundler-cache: true
+    - name: show ruby version
+      run: ruby -v
+    - name: bundle install
+      run: bundle install
+    #
+    # This step avoids the following error in the JRuby 9.4 test.
+    #
+    # Gem::LoadError: You have already activated rake 13.0.6,
+    # but your Gemfile requires rake 13.1.0. Prepending
+    # `bundle exec` to your command may solve this.
+    #
+    - name: install rake 13.1.0
+      run: gem install rake -v 13.1.0
+    - name: install embulk.jar
+      run: "curl -L -o embulk.jar https://github.com/embulk/embulk/releases/download/v0.10.49/embulk-0.10.49.jar"
+    - name: rake test
+      run: bundle exec env RUBYOPT="-r ./embulk.jar" rake test

data/.github/workflows/publish.yml

@@ -0,0 +1,41 @@
+name: Publish
+on:
+  push:
+    tags:
+      - "v0.*"
+jobs:
+  publish:
+    runs-on: ubuntu-latest
+    environment: maven-central-and-ruby-gems
+    strategy:
+      fail-fast: true
+    steps:
+    - uses: actions/checkout@v4
+    - name: Set up Ruby
+      uses: ruby/setup-ruby@v1
+      with:
+        ruby-version: 3.3.0
+    # get tag variable using {{ github.ref_name }}
+    #
+    # References:
+    # * https://docs.github.com/en/actions/learn-github-actions/contexts#github-context
+    # * https://docs.github.com/en/actions/learn-github-actions/variables#default-environment-variables
+    - name: extract gem version from tag
+      id: vars
+      run: echo version=${{ github.ref_name }} | sed -e 's/v0/0/' >> $GITHUB_OUTPUT
+    #
+    # From gem push documents.
+    #
+    # The push command will use ~/.gem/credentials to authenticate to a server,
+    # but you can use the RubyGems environment variable GEM_HOST_API_KEY
+    # to set the api key to authenticate.
+    #
+    # https://guides.rubygems.org/command-reference/#gem-push
+    #
+    - name: Publish
+      run: |
+        rake build
+        gem push pkg/${EMBULK_PLUGIN_NAME}-${{ steps.vars.outputs.version }}.gem
+      env:
+        EMBULK_PLUGIN_NAME: embulk-output-bigquery
+        GEM_HOST_API_KEY: "${{secrets.RUBYGEMS_API_KEY}}"

data/CHANGELOG.md

@@ -1,3 +1,9 @@
+## 0.7.0 - 2024-02-1
+* [enhancement] Add support Embulk 0.11.x
+
+## 0.6.9 - 2023-03-16
+* [enhancement] Add SSLException to retry job (thanks to @mzumi)
+
 ## 0.6.8 - 2022-10-12
 * [enhancement] Support JSON type (thanks to @civitaspo )
 * [maintenance] Add an error message in order to retry (thanks to @mzumi)

data/Gemfile

@@ -1,8 +1,7 @@
 source 'https://rubygems.org/'
 
 gemspec
-gem 'embulk', '< 0.10'
-gem 'liquid', '= 4.0.0' # the version included in embulk.jar
+gem 'embulk', '= 0.10.49'
 gem 'embulk-parser-none'
 gem 'embulk-parser-jsonl'
 gem 'pry-nav'

data/README.md

@@ -1,7 +1,5 @@
 # embulk-output-bigquery
 
-[![Build Status](https://secure.travis-ci.org/embulk/embulk-output-bigquery.png?branch=master)](http://travis-ci.org/embulk/embulk-output-bigquery)
-
 [Embulk](https://github.com/embulk/embulk/) output plugin to load/insert data into [Google BigQuery](https://cloud.google.com/bigquery/) using [direct insert](https://cloud.google.com/bigquery/loading-data-into-bigquery#loaddatapostrequest)
 
 ## Overview
@@ -14,6 +12,13 @@ https://developers.google.com/bigquery/loading-data-into-bigquery
 * **Cleanup supported**: no
 * **Dynamic table creating**: yes
 
+### Supported Embulk
+
+| gem version      | Embulk version     |
+|------------------|--------------------|
+| 0.7.0 and higher | v0.11.0 and higher |
+| 0.6.9 and lower  | v0.9.X and lower   |
+
 ### NOT IMPLEMENTED
 * insert data over streaming inserts
   * for continuous real-time insertions

data/embulk-output-bigquery.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |spec|
   spec.name          = "embulk-output-bigquery"
-  spec.version       = "0.6.8"
+  spec.version       = "0.7.0"
   spec.authors       = ["Satoshi Akama", "Naotoshi Seo"]
   spec.summary       = "Google BigQuery output plugin for Embulk"
   spec.description   = "Embulk plugin that insert records to Google BigQuery."
@@ -14,18 +14,13 @@ Gem::Specification.new do |spec|
   spec.test_files    = spec.files.grep(%r{^(test|spec)/})
   spec.require_paths = ["lib"]
 
-  # TODO
-  # signet 0.12.0 and google-api-client 0.33.0 require >= Ruby 2.4.
-  # Embulk 0.9 use JRuby 9.1.X.Y and it's compatible with Ruby 2.3.
-  # So, force install signet < 0.12 and google-api-client < 0.33.0
-  # Also, representable version >= 3.1.0 requires Ruby version >= 2.4
-  spec.add_dependency 'signet', '~> 0.7', '< 0.12.0'
-  spec.add_dependency 'google-api-client','< 0.33.0'
+  # the latest version
+  spec.add_dependency 'google-api-client','= 0.53.0'
   spec.add_dependency 'time_with_zone'
-  spec.add_dependency "representable", ['~> 3.0.0', '< 3.1']
-  # faraday 1.1.0 require >= Ruby 2.4.
-  # googleauth 0.9.0 requires faraday ~> 0.12
-  spec.add_dependency "faraday", '~> 0.12'
+  spec.add_dependency 'thwait'
+  # activesupport require Ruby >= 2.7.0
+  # jruby-9.3.0.0 is MRI 2.6 compatible
+  spec.add_dependency 'activesupport', "< 7.0"
 
   spec.add_development_dependency 'bundler', ['>= 1.10.6']
   spec.add_development_dependency 'rake', ['>= 10.0']

data/lib/embulk/output/bigquery/bigquery_client.rb

@@ -121,7 +121,7 @@ module Embulk
               opts = {}
 
               Embulk.logger.debug { "embulk-output-bigquery: insert_job(#{@project}, #{body}, #{opts})" }
-              response = with_network_retry { client.insert_job(@project, body, opts) }
+              response = with_network_retry { client.insert_job(@project, body, **opts) }
               unless @task['is_skip_job_result_check']
                 response = wait_load('Load', response)
               end
@@ -222,7 +222,7 @@ module Embulk
                 # },
               }
               Embulk.logger.debug { "embulk-output-bigquery: insert_job(#{@project}, #{body}, #{opts})" }
-              response = with_network_retry { client.insert_job(@project, body, opts) }
+              response = with_network_retry { client.insert_job(@project, body, **opts) }
               if @task['is_skip_job_result_check']
                 response
               else
@@ -278,7 +278,7 @@ module Embulk
 
               opts = {}
               Embulk.logger.debug { "embulk-output-bigquery: insert_job(#{@project}, #{body}, #{opts})" }
-              response = with_network_retry { client.insert_job(@project, body, opts) }
+              response = with_network_retry { client.insert_job(@project, body, **opts) }
               wait_load('Copy', response)
             rescue Google::Apis::ServerError, Google::Apis::ClientError, Google::Apis::AuthorizationError => e
               response = {status_code: e.status_code, message: e.message, error_class: e.class}
@@ -372,7 +372,7 @@ module Embulk
             end
             opts = {}
             Embulk.logger.debug { "embulk-output-bigquery: insert_dataset(#{@project}, #{dataset}, #{@location_for_log}, #{body}, #{opts})" }
-            with_network_retry { client.insert_dataset(@project, body, opts) }
+            with_network_retry { client.insert_dataset(@project, body, **opts) }
           rescue Google::Apis::ServerError, Google::Apis::ClientError, Google::Apis::AuthorizationError => e
             if e.status_code == 409 && /Already Exists:/ =~ e.message
               # ignore 'Already Exists' error
@@ -447,7 +447,7 @@ module Embulk
 
             opts = {}
             Embulk.logger.debug { "embulk-output-bigquery: insert_table(#{@project}, #{dataset}, #{@location_for_log}, #{body}, #{opts})" }
-            with_network_retry { client.insert_table(@project, dataset, body, opts) }
+            with_network_retry { client.insert_table(@project, dataset, body, **opts) }
           rescue Google::Apis::ServerError, Google::Apis::ClientError, Google::Apis::AuthorizationError => e
             if e.status_code == 409 && /Already Exists:/ =~ e.message
               # ignore 'Already Exists' error

data/lib/embulk/output/bigquery/google_client.rb

@@ -49,7 +49,7 @@ module Embulk
           retries = 0
           begin
             yield
-          rescue ::Java::Java.net.SocketException, ::Java::Java.net.ConnectException => e
+          rescue ::Java::Java.net.SocketException, ::Java::Java.net.ConnectException, ::Java::JavaxNetSsl::SSLException => e
             retry_messages = [
               'Broken pipe',
               'Connection reset',

data/test/helper.rb

@@ -4,14 +4,14 @@ require 'bundler/setup'
 require 'test/unit'
 require 'test/unit/rr'
 
+# Embulk 0.10.x introduced new bootstrap mechanism.
+# https://github.com/embulk/embulk/blob/641f35fec064cca7b1a7314d634a4b64ef8637f1/embulk-ruby/test/vanilla/run-test.rb#L8-L13
+static_initializer = Java::org.embulk.EmbulkDependencyClassLoader.staticInitializer().useSelfContainedJarFiles()
+static_initializer.java_send :initialize
+
+require 'embulk/java/bootstrap'
 require 'embulk'
-begin
-  # Embulk ~> 0.8.x
-  Embulk.setup
-rescue NotImplementedError, NoMethodError, NameError
-  # Embulk ~> 0.9.x
-  require 'embulk/java/bootstrap'
-end
+
 Embulk.logger = Embulk::Logger.new('/dev/null')
 
 APP_ROOT = File.expand_path('../', __dir__)

data/test/test_transaction.rb

@@ -109,7 +109,7 @@ module Embulk
           task = Bigquery.configure(config, schema, processor_count)
           any_instance_of(BigqueryClient) do |obj|
             mock(obj).get_dataset(config['dataset'])
-            mock(obj).create_table_if_not_exists(config['temp_table'])
+            mock(obj).create_table_if_not_exists(config['temp_table'], options: {"expiration_time"=>nil})
             mock(obj).create_table_if_not_exists(config['table'])
             mock(obj).copy(config['temp_table'], config['table'], write_disposition: 'WRITE_TRUNCATE')
             mock(obj).delete_table(config['temp_table'])
@@ -122,7 +122,7 @@ module Embulk
           task = Bigquery.configure(config, schema, processor_count)
           any_instance_of(BigqueryClient) do |obj|
             mock(obj).get_dataset(config['dataset'])
-            mock(obj).create_table_if_not_exists(config['temp_table'])
+            mock(obj).create_table_if_not_exists(config['temp_table'], options: {"expiration_time"=>nil})
             mock(obj).create_table_if_not_exists(config['table'])
             mock(obj).copy(config['temp_table'], config['table'], write_disposition: 'WRITE_TRUNCATE')
             mock(obj).delete_table(config['temp_table'])
@@ -138,7 +138,7 @@ module Embulk
           any_instance_of(BigqueryClient) do |obj|
             mock(obj).get_dataset(config['dataset'])
             mock(obj).get_dataset(config['dataset_old'])
-            mock(obj).create_table_if_not_exists(config['temp_table'])
+            mock(obj).create_table_if_not_exists(config['temp_table'], options: {"expiration_time"=>nil})
             mock(obj).create_table_if_not_exists(config['table'])
             mock(obj).create_table_if_not_exists(config['table_old'], dataset: config['dataset_old'])
 
@@ -158,7 +158,7 @@ module Embulk
             mock(obj).create_dataset(config['dataset'])
             mock(obj).create_dataset(config['dataset_old'], reference: config['dataset'])
             mock(obj).create_table_if_not_exists(config['table'])
-            mock(obj).create_table_if_not_exists(config['temp_table'])
+            mock(obj).create_table_if_not_exists(config['temp_table'], options: {"expiration_time"=>nil})
             mock(obj).create_table_if_not_exists(config['table_old'], dataset: config['dataset_old'])
 
             mock(obj).get_table_or_partition(config['table'])
@@ -176,7 +176,7 @@ module Embulk
           any_instance_of(BigqueryClient) do |obj|
             mock(obj).get_dataset(config['dataset'])
             mock(obj).get_dataset(config['dataset_old'])
-            mock(obj).create_table_if_not_exists(config['temp_table'])
+            mock(obj).create_table_if_not_exists(config['temp_table'], options: {"expiration_time"=>nil})
             mock(obj).create_table_if_not_exists(config['table'])
             mock(obj).create_table_if_not_exists(config['table_old'], dataset: config['dataset_old'])
 
@@ -196,7 +196,7 @@ module Embulk
           task = Bigquery.configure(config, schema, processor_count)
           any_instance_of(BigqueryClient) do |obj|
             mock(obj).get_dataset(config['dataset'])
-            mock(obj).create_table_if_not_exists(config['temp_table'])
+            mock(obj).create_table_if_not_exists(config['temp_table'], options: {"expiration_time"=>nil})
             mock(obj).create_table_if_not_exists(config['table'])
             mock(obj).copy(config['temp_table'], config['table'], write_disposition: 'WRITE_APPEND')
             mock(obj).delete_table(config['temp_table'])
@@ -209,7 +209,7 @@ module Embulk
           task = Bigquery.configure(config, schema, processor_count)
           any_instance_of(BigqueryClient) do |obj|
             mock(obj).get_dataset(config['dataset'])
-            mock(obj).create_table_if_not_exists(config['temp_table'])
+            mock(obj).create_table_if_not_exists(config['temp_table'], options: {"expiration_time"=>nil})
             mock(obj).create_table_if_not_exists(config['table'])
             mock(obj).copy(config['temp_table'], config['table'], write_disposition: 'WRITE_APPEND')
             mock(obj).delete_table(config['temp_table'])

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: embulk-output-bigquery
 version: !ruby/object:Gem::Version
-  version: 0.6.8
+  version: 0.7.0
 platform: ruby
 authors:
 - Satoshi Akama
@@ -9,42 +9,22 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2022-10-12 00:00:00.000000000 Z
+date: 2024-02-13 00:00:00.000000000 Z
 dependencies:
-- !ruby/object:Gem::Dependency
-  name: signet
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '0.7'
-    - - "<"
-      - !ruby/object:Gem::Version
-        version: 0.12.0
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '0.7'
-    - - "<"
-      - !ruby/object:Gem::Version
-        version: 0.12.0
 - !ruby/object:Gem::Dependency
   name: google-api-client
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "<"
+    - - '='
       - !ruby/object:Gem::Version
-        version: 0.33.0
+        version: 0.53.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "<"
+    - - '='
       - !ruby/object:Gem::Version
-        version: 0.33.0
+        version: 0.53.0
 - !ruby/object:Gem::Dependency
   name: time_with_zone
   requirement: !ruby/object:Gem::Requirement
@@ -60,39 +40,33 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: representable
+  name: thwait
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: 3.0.0
-    - - "<"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '3.1'
+        version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: 3.0.0
-    - - "<"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '3.1'
+        version: '0'
 - !ruby/object:Gem::Dependency
-  name: faraday
+  name: activesupport
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - "<"
       - !ruby/object:Gem::Version
-        version: '0.12'
+        version: '7.0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - "<"
       - !ruby/object:Gem::Version
-        version: '0.12'
+        version: '7.0'
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
@@ -129,6 +103,8 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
+- ".github/workflows/check.yml"
+- ".github/workflows/publish.yml"
 - ".gitignore"
 - ".travis.yml"
 - CHANGELOG.md
@@ -172,7 +148,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.3.7
+rubygems_version: 3.5.3
 signing_key: 
 specification_version: 4
 summary: Google BigQuery output plugin for Embulk