bigquery_migration 0.3.0 → 0.3.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: 97e77d9eca83b83c064a4e5328dbd3d776633290
-  data.tar.gz: 8de9b03871570d5558e29a16d84b0fbef6873fc7
+SHA256:
+  metadata.gz: 68a48ec86057ec0242c55beb90f6dfc8db688e6073d634fbcc9a03e0066b3357
+  data.tar.gz: 2a4358172e12aea071e5b260aa9d8f2dd57ec4d9a29d01ba2926f563ab58beb4
 SHA512:
-  metadata.gz: 7de67b97655404b7f4a9f209484932252ed9d64da3e5535830012ce22cf6fcb3ed1283f13e8859017072bcaff43627d069f9b03851d97b00157c262b82351468
-  data.tar.gz: f82e01cb2b9d25c24344dd839d56807039c7c24fdf4ea055b308a02b84feecb49997ce73c7faf698b6c6345a9c3435350be8fb416d2a53894918fad6836c502e
+  metadata.gz: 1df093bcd1f22dc01cd6e09e8ed10d6ad7b85182ecb0d268b8afac48d46489eb5ed7c1f5ff3d5f564b0d7f15017aaf7b02f55fa1bef23618d4f4fde044bb5f67
+  data.tar.gz: 9af8c78d3ed6c9dfd3a269bde24e016f7cf4b63b2ac8a3e2f39ac513a1e035657329c8709ed7ee8ef0f71f5c91881d330804a5af7652bf8b54e0338c50ad24c2

data/CHANGELOG.md

@@ -1,3 +1,9 @@
+# 0.3.1 (2018/05/23)
+
+Enhancements:
+
+* Support newly added location option of google-api-ruby-client.
+
 # 0.3.0 (2017/04/26)
 
 Enhancements:

data/README.md

@@ -2,6 +2,10 @@
 
 BigqueryMigraiton is a tool or a ruby library to migrate (or alter) BigQuery table schema.
 
+## Requirements
+
+* Ruby >= 2.3.0
+
 ## Installation
 
 Add this line to your application's Gemfile:
@@ -31,6 +35,8 @@ bigquery: &bigquery
   json_keyfile: your-project-000.json
   dataset: your_dataset_name
   table: your_table_name
+  # If your data is in a location other than the US or EU multi-region, you must specify the location
+  # location: asia-northeast1
 
 actions:
 - action: create_dataset
@@ -59,9 +65,12 @@ $ bundle exec bq_migrate run config.yml --exec
 require 'bigquery_migration'
 
 config = {
-  json_keyfile: '/path/to/your-project-000.json'
-  dataset: 'your_dataset_name'
-  table: 'your_table_name'
+  json_keyfile: '/path/to/your-project-000.json',
+  dataset: 'your_dataset_name',
+  table: 'your_table_name',
+
+  # If your data is in a location other than the US or EU multi-region, you must specify the location
+  # location: asia-northeast1,
 }
 columns = [
   { name: 'string', type: 'STRING' },

data/bigquery_migration.gemspec

@@ -6,8 +6,8 @@ require 'bigquery_migration/version'
 Gem::Specification.new do |spec|
   spec.name          = "bigquery_migration"
   spec.version       = BigqueryMigration::VERSION
-  spec.authors       = ["Naotoshi Seo"]
-  spec.email         = ["sonots@gmail.com"]
+  spec.authors       = ["Naotoshi Seo", "kysnm", "potato2003"]
+  spec.email         = ["sonots@gmail.com", "tokyoincidents.g@gmail.com", "potato2003@gmail.com"]
 
   spec.summary       = %q{Migrate BigQuery table schema}
   spec.description   = %q{Migrate BigQuery table schema.}

data/lib/bigquery_migration/bigquery_wrapper.rb

@@ -30,7 +30,7 @@ class BigqueryMigration
       client.request_options.retries = retries
       client.client_options.open_timeout_sec = open_timeout_sec
       if client.request_options.respond_to?(:timeout_sec)
-        client.client_options.timeout_sec = timeout_sec
+        client.request_options.timeout_sec = timeout_sec
       else # google-api-ruby-client >= v0.11.0
         if timeout_sec
           logger.warn { "timeout_sec is deprecated in google-api-ruby-client >= v0.11.0. Use read_timeout_sec instead" }
@@ -49,9 +49,9 @@ class BigqueryMigration
           token_credential_uri: "https://accounts.google.com/o/oauth2/token",
           audience: "https://accounts.google.com/o/oauth2/token",
           scope: scope,
-          client_id:     credentials['client_id'],
-          client_secret: credentials['client_secret'],
-          refresh_token: credentials['refresh_token']
+          client_id:     credentials[:client_id],
+          client_secret: credentials[:client_secret],
+          refresh_token: credentials[:refresh_token]
         )
         auth.refresh!
       when 'compute_engine'
@@ -116,7 +116,9 @@ class BigqueryMigration
             dataset_id: dataset,
           },
         }.merge(hint)
+        body[:location] = location if location
         opts = {}
+
         logger.debug { "#{head}insert_dataset(#{project}, #{body}, #{opts})" }
         unless dry_run?
           response = client.insert_dataset(project, body, opts)
@@ -479,6 +481,7 @@ class BigqueryMigration
           }
         }
       }
+      body[:job_reference][:location] = location if location
       opts = {}
 
       logger.info  { "#{head}insert_job(#{project}, #{body}, #{opts})" }
@@ -519,6 +522,7 @@ class BigqueryMigration
           }
         }
       }
+      body[:job_reference][:location] = location if location
       opts = {}
 
       logger.info { "#{head}insert_job(#{project}, #{body}, #{opts})" }
@@ -563,7 +567,11 @@ class BigqueryMigration
             "job_id:[#{job_id}] elapsed_time:#{elapsed.to_f}sec status:[#{status}]"
           }
           sleep wait_interval
-          _response = client.get_job(project, job_id)
+          if support_location_keyword?
+            _response = client.get_job(project, job_id, location: location)
+          else
+            _response = client.get_job(project, job_id)
+          end
         end
       end
 
@@ -707,6 +715,11 @@ class BigqueryMigration
       result.merge!( before_columns: before_columns, after_columns: after_columns )
     end
 
+    # the location keyword arguments are available in google-api-client v0.19.6 or later
+    def support_location_keyword?
+      @support_location_keyword ||= client.method(:get_job).parameters.include?([:key, :location])
+    end
+
     # For old version compatibility
     # Use credentials_file or credentials instead
     def json_key
@@ -714,7 +727,7 @@ class BigqueryMigration
         begin
           case json_keyfile
           when String
-            return JSON.parse(File.read(json_keyfile))
+            return HashUtil.deep_symbolize_keys(JSON.parse(File.read(json_keyfile)))
           when Hash
             return json_keyfile[:content]
           else
@@ -729,11 +742,11 @@ class BigqueryMigration
 
     # compute_engine, authorized_user, service_account
     def auth_method
-      @auth_method ||= ENV['AUTH_METHOD'] || config.fetch(:auth_method, nil) || credentials['type'] || 'compute_engine'
+      @auth_method ||= ENV['AUTH_METHOD'] || config.fetch(:auth_method, nil) || credentials[:type] || 'compute_engine'
     end
 
     def credentials
-      json_key || JSON.parse(config.fetch(:credentials, nil) || File.read(credentials_file))
+      json_key || HashUtil.deep_symbolize_keys(JSON.parse(config.fetch(:credentials, nil) || File.read(credentials_file)))
     end
 
     def credentials_file
@@ -775,7 +788,7 @@ class BigqueryMigration
     end
 
     def service_account
-      @service_account ||= ENV['GOOGLE_SERVICE_ACCOUNT'] || config.fetch(:service_account, nil) || credentials['client_email'] || service_account_default
+      @service_account ||= ENV['GOOGLE_SERVICE_ACCOUNT'] || config.fetch(:service_account, nil) || credentials[:client_email] || service_account_default
     end
 
     def retries
@@ -800,8 +813,8 @@ class BigqueryMigration
     end
 
     def project
-      @project ||= ENV['GOOGLE_PROJECT'] || config.fetch(:project, nil) || credentials['project_id']
-      @project ||= credentials['client_email'].chomp('.iam.gserviceaccount.com').split('@').last if credentials['client_email']
+      @project ||= ENV['GOOGLE_PROJECT'] || config.fetch(:project, nil) || credentials[:project_id]
+      @project ||= credentials[:client_email].chomp('.iam.gserviceaccount.com').split('@').last if credentials[:client_email]
       @project ||= project_default || raise(ConfigError, '`project` is required.')
     end
 
@@ -813,6 +826,10 @@ class BigqueryMigration
       @table  ||= config[:table]   || raise(ConfigError, '`table` is required.')
     end
 
+    def location
+      config[:location]
+    end
+
     def job_status_polling_interval
       @job_status_polling_interval ||= config[:job_status_polling_interval] || 5
     end

data/lib/bigquery_migration/version.rb

@@ -1,3 +1,3 @@
 class BigqueryMigration
-  VERSION = "0.3.0"
+  VERSION = "0.3.1"
 end

metadata

@@ -1,14 +1,16 @@
 --- !ruby/object:Gem::Specification
 name: bigquery_migration
 version: !ruby/object:Gem::Version
-  version: 0.3.0
+  version: 0.3.1
 platform: ruby
 authors:
 - Naotoshi Seo
+- kysnm
+- potato2003
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2017-04-26 00:00:00.000000000 Z
+date: 2018-05-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: google-api-client
@@ -153,6 +155,8 @@ dependencies:
 description: Migrate BigQuery table schema.
 email:
 - sonots@gmail.com
+- tokyoincidents.g@gmail.com
+- potato2003@gmail.com
 executables:
 - bq-migrate
 - bq_migrate
@@ -214,7 +218,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.6.11
+rubygems_version: 2.7.4
 signing_key: 
 specification_version: 4
 summary: Migrate BigQuery table schema