bigquery_migration 0.2.2 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 4cc3b6f5ad69666d2dec7c19ba97cbafecd8f74a
-  data.tar.gz: 490f4b4c6dd4801771de9296d9907a90ea5f43b2
+  metadata.gz: 97e77d9eca83b83c064a4e5328dbd3d776633290
+  data.tar.gz: 8de9b03871570d5558e29a16d84b0fbef6873fc7
 SHA512:
-  metadata.gz: b486f63c1e1119fc8061308d073559d88cf006dcb935c73dd023b96ae2b783a58ef7f4f2f4717ed30e72f6536c8c46335430f0e0fa993343fbf1781397ef3e61
-  data.tar.gz: 2c36c1a2a889d7a8137139bd6ec1ca1900b2e5d8129ab6b3a073c206b5ca2831cd3fbb912c8b16fe84302baba63a6c0b90e607c7da7dc8aa7e301763fa22dfc6
+  metadata.gz: 7de67b97655404b7f4a9f209484932252ed9d64da3e5535830012ce22cf6fcb3ed1283f13e8859017072bcaff43627d069f9b03851d97b00157c262b82351468
+  data.tar.gz: f82e01cb2b9d25c24344dd839d56807039c7c24fdf4ea055b308a02b84feecb49997ce73c7faf698b6c6345a9c3435350be8fb416d2a53894918fad6836c502e

data/CHANGELOG.md

@@ -1,3 +1,9 @@
+# 0.3.0 (2017/04/26)
+
+Enhancements:
+
+* Support more authentication methods such as oauth, compute_engine, application_default
+
 # 0.2.2 (2017/04/04)
 
 Enhancements:

data/README.md

@@ -95,13 +95,40 @@ This tool has an advantage that it is **faster** than reloading data entirely.
 
 ### Run example:
 
-Prepare `example/your-project-000.json`, then
+**Service Account**
+
+Prepare your service account json at `example/your-project-000.json`, then
 
 ```
 $ bundle exec bq_migrate run example/example.yml # dry-run
 $ bundle exec bq_migrate run example/example.yml --exec
 ```
 
+**OAuth**
+
+Install gcloud into your development environment:
+
+```
+curl https://sdk.cloud.google.com | bash
+gcloud init
+gcloud auth login
+gcloud auth application-default login
+gcloud config set project <GCP_PROJECT_NAME>
+```
+
+Make sure `gcloud` works
+
+```
+gcloud compute instances list
+```
+
+Run as:
+
+```
+$ bundle exec bq_migrate run example/application_default.yml # dry-run
+$ bundle exec bq_migrate run example/application_default.yml --exec
+```
+
 ### Run test:
 
 ```
@@ -110,7 +137,7 @@ $ bundle exec rake test
 
 To run tests which directly connects to BigQuery, prepare `example/your-project-000.json`, then
 
-````
+```
 $ bundle exec rake test
 ```
 

data/bigquery_migration.gemspec

@@ -22,6 +22,7 @@ Gem::Specification.new do |spec|
   spec.add_dependency "google-api-client"
   spec.add_dependency "tzinfo"
   spec.add_dependency "thor"
+  spec.add_dependency "inifile"
 
   spec.add_development_dependency "bundler", "~> 1.11"
   spec.add_development_dependency "rake", "~> 10.0"

data/example/application_default.yml

@@ -0,0 +1,23 @@
+bigquery: &bigquery
+  # project: read from ~/.config/gcloud/configurations/config_default
+  dataset: your_dataset_name
+  table: your_table_name
+
+actions:
+- action: create_dataset
+  <<: *bigquery
+- action: migrate_table
+  <<: *bigquery
+  columns:
+    - { name: 'timestamp', type: 'TIMESTAMP' }
+    - name: 'record'
+      type: 'RECORD'
+      fields:
+        - { name: 'string', type: 'STRING' }
+        - { name: 'integer', type: 'INTEGER' }
+        - { name: 'bytes', type: 'BYTES' }
+- action: migrate_table
+  <<: *bigquery
+  schema_file: example/schema.json
+- action: delete_table
+  <<: *bigquery

data/example/copy_table.yml

@@ -1,5 +1,5 @@
 bigquery: &bigquery
-  json_keyfile: example/your-project-000.json
+  credentials_file: example/your-project-000.json
   dataset: your_dataset_name
   table: your_table_name
 

data/example/example.yml

@@ -1 +1 @@
-migrate_table.yml
+example/migrate_table.yml

data/example/insert_select.yml

@@ -1,5 +1,5 @@
 bigquery: &bigquery
-  json_keyfile: example/your-project-000.json
+  credentials_file: example/your-project-000.json
   dataset: your_dataset_name
   table: your_table_name
 

data/example/migrate_partitioned_table.yml

@@ -1,5 +1,5 @@
 bigquery: &bigquery
-  json_keyfile: example/your-project-000.json
+  credentials_file: example/your-project-000.json
   dataset: your_dataset_name
   table: your_table_name
 

data/example/migrate_table.yml

@@ -1,5 +1,5 @@
 bigquery: &bigquery
-  json_keyfile: example/your-project-000.json
+  credentials_file: example/your-project-000.json
   dataset: your_dataset_name
   table: your_table_name
 

data/example/table_info.yml

@@ -1,5 +1,5 @@
 bigquery: &bigquery
-  json_keyfile: example/your-project-000.json
+  credentials_file: example/your-project-000.json
   dataset: your_dataset_name
   table: your_table_name
 

data/exe/bq-migrate

@@ -1 +1 @@
-bq_migrate
+exe/bq_migrate

data/lib/bigquery_migration/bigquery_wrapper.rb

@@ -8,6 +8,7 @@ require_relative 'hash_util'
 require 'google/apis/bigquery_v2'
 require 'google/api_client/auth/key_utils'
 require 'securerandom'
+require 'inifile'
 
 class BigqueryMigration
   class BigqueryWrapper
@@ -20,92 +21,50 @@ class BigqueryMigration
     def initialize(config, opts = {})
       @config = HashUtil.deep_symbolize_keys(config)
       @opts = HashUtil.deep_symbolize_keys(opts)
-      configure
-    end
-
-    def configure
-      if json_keyfile = config[:json_keyfile]
-        json_key =
-          case json_keyfile
-          when String
-            File.read(json_keyfile)
-          when Hash
-            json_keyfile[:content]
-          else
-            raise ConfigError.new "Unsupported json_keyfile type"
-          end
-        json_keyparams =
-          begin
-            case json_key
-            when String
-              HashUtil.deep_symbolize_keys(JSON.parse(json_key))
-            when Hash
-              HashUtil.deep_symbolize_keys(json_key)
-            end
-          rescue => e
-            raise ConfigError.new "json_keyfile is not a JSON file"
-          end
-      end
-
-      if json_keyparams
-        config[:project] ||= json_keyparams[:project_id]
-        config[:json_key] = json_keyparams.to_json
-      end
-
-      config[:retries] ||= 5
-    end
-
-    def project
-      @project ||= config[:project] || raise(ConfigError, '`project` is required.')
-    end
-
-    def dataset
-      @dataset ||= config[:dataset] || raise(ConfigError, '`dataset` is required.')
-    end
-
-    def table
-      @table  ||= config[:table]   || raise(ConfigError, '`table` is required.')
-    end
-
-    def job_status_polling_interval
-      @job_status_polling_interval ||= config[:job_status_polling_interval] || 5
-    end
-
-    def job_status_max_polling_time
-      @job_status_max_polling_time ||= config[:job_status_polling_time] || 3600
-    end
-
-    def dry_run?
-      @opts[:dry_run]
-    end
-
-    def head
-      dry_run? ? '(DRY-RUN) ' : '(EXECUTE) '
     end
 
     def client
       return @cached_client if @cached_client && @cached_client_expiration > Time.now
 
       client = Google::Apis::BigqueryV2::BigqueryService.new
-      client.request_options.retries = config[:retries]
+      client.request_options.retries = retries
+      client.client_options.open_timeout_sec = open_timeout_sec
       if client.request_options.respond_to?(:timeout_sec)
-        client.request_options.open_timeout_sec = config[:open_timeout_sec] || 300
-        client.request_options.timeout_sec = config[:timeout_sec] || 300
+        client.client_options.timeout_sec = timeout_sec
       else # google-api-ruby-client >= v0.11.0
-        if config[:timeout_sec]
+        if timeout_sec
           logger.warn { "timeout_sec is deprecated in google-api-ruby-client >= v0.11.0. Use read_timeout_sec instead" }
         end
-        client.client_options.open_timeout_sec = config[:open_timeout_sec] || 300 # default: 60
-        client.client_options.send_timeout_sec = config[:send_timeout_sec] || 300 # default: 120
-        client.client_options.read_timeout_sec = config[:read_timeout_sec] || config[:timeout_sec] || 300 # default: 60
+        client.client_options.send_timeout_sec = send_timeout_sec
+        client.client_options.read_timeout_sec = read_timeout_sec
       end
       logger.debug { "client_options: #{client.client_options.to_h}" }
       logger.debug { "request_options: #{client.request_options.to_h}" }
 
       scope = "https://www.googleapis.com/auth/bigquery"
 
-      key = StringIO.new(config[:json_key])
-      auth = Google::Auth::ServiceAccountCredentials.make_creds(json_key_io: key, scope: scope)
+      case auth_method
+      when 'authorized_user'
+        auth = Signet::OAuth2::Client.new(
+          token_credential_uri: "https://accounts.google.com/o/oauth2/token",
+          audience: "https://accounts.google.com/o/oauth2/token",
+          scope: scope,
+          client_id:     credentials['client_id'],
+          client_secret: credentials['client_secret'],
+          refresh_token: credentials['refresh_token']
+        )
+        auth.refresh!
+      when 'compute_engine'
+        auth = Google::Auth::GCECredentials.new
+      when 'service_account'
+        key = StringIO.new(credentials.to_json)
+        auth = Google::Auth::ServiceAccountCredentials.make_creds(json_key_io: key, scope: scope)
+      when 'application_default'
+        auth = Google::Auth.get_application_default([scope])
+      else
+        raise ConfigError, "Unknown auth method: #{auth_method}"
+      end
+
       client.authorization = auth
 
       @cached_client_expiration = Time.now + 1800
@@ -747,5 +706,127 @@ class BigqueryMigration
 
       result.merge!( before_columns: before_columns, after_columns: after_columns )
     end
+
+    # For old version compatibility
+    # Use credentials_file or credentials instead
+    def json_key
+      if json_keyfile = config[:json_keyfile]
+        begin
+          case json_keyfile
+          when String
+            return JSON.parse(File.read(json_keyfile))
+          when Hash
+            return json_keyfile[:content]
+          else
+            raise ConfigError.new "Unsupported json_keyfile type"
+          end
+        rescue => e
+          raise ConfigError.new "json_keyfile is not a JSON file"
+        end
+      end
+      nil
+    end
+
+    # compute_engine, authorized_user, service_account
+    def auth_method
+      @auth_method ||= ENV['AUTH_METHOD'] || config.fetch(:auth_method, nil) || credentials['type'] || 'compute_engine'
+    end
+
+    def credentials
+      json_key || JSON.parse(config.fetch(:credentials, nil) || File.read(credentials_file))
+    end
+
+    def credentials_file
+      @credentials_file ||= File.expand_path(
+        # ref. https://developers.google.com/identity/protocols/application-default-credentials
+        ENV['GOOGLE_APPLICATION_CREDENTIALS'] ||
+        config.fetch(:credentials_file, nil) ||
+        (File.exist?(global_application_default_credentials_file) ? global_application_default_credentials_file : application_default_credentials_file)
+      )
+    end
+
+    def application_default_credentials_file
+      @application_default_credentials_file ||= File.expand_path("~/.config/gcloud/application_default_credentials.json")
+    end
+
+    def global_application_default_credentials_file
+      @global_application_default_credentials_file ||= '/etc/google/auth/application_default_credentials.json'
+    end
+
+    def config_default_file
+      File.expand_path('~/.config/gcloud/configurations/config_default')
+    end
+
+    def config_default
+      # {core:{account:'xxx',project:'xxx'},compute:{zone:'xxx}}
+      @config_default ||= File.readable?(config_default_file) ? HashUtil.deep_symbolize_keys(IniFile.load(config_default_file).to_h) : {}
+    end
+
+    def service_account_default
+      (config_default[:core] || {})[:account]
+    end
+
+    def project_default
+      (config_default[:core] || {})[:project]
+    end
+
+    def zone_default
+      (config_default[:compute] || {})[:zone]
+    end
+
+    def service_account
+      @service_account ||= ENV['GOOGLE_SERVICE_ACCOUNT'] || config.fetch(:service_account, nil) || credentials['client_email'] || service_account_default
+    end
+
+    def retries
+      @retries ||= ENV['RETRIES'] || config.fetch(:retries, nil) || 5
+    end
+
+    # For google-api-client < 0.11.0. Deprecated
+    def timeout_sec
+      @timeout_sec ||= ENV['TIMEOUT_SEC'] || config.fetch(:timeout_sec, nil)
+    end
+
+    def send_timeout_sec
+      @send_timeout_sec ||= ENV['SEND_TIMEOUT_SEC'] || config.fetch(:send_timeout_sec, nil) || 60
+    end
+
+    def read_timeout_sec
+      @read_timeout_sec ||= ENV['READ_TIMEOUT_SEC'] || config.fetch(:read_timeout_sec, nil) || timeout_sec || 300
+    end
+
+    def open_timeout_sec
+      @open_timeout_sec ||= ENV['OPEN_TIMEOUT_SEC'] || config.fetch(:open_timeout_sec, nil) || 300
+    end
+
+    def project
+      @project ||= ENV['GOOGLE_PROJECT'] || config.fetch(:project, nil) || credentials['project_id']
+      @project ||= credentials['client_email'].chomp('.iam.gserviceaccount.com').split('@').last if credentials['client_email']
+      @project ||= project_default || raise(ConfigError, '`project` is required.')
+    end
+
+    def dataset
+      @dataset ||= config[:dataset] || raise(ConfigError, '`dataset` is required.')
+    end
+
+    def table
+      @table  ||= config[:table]   || raise(ConfigError, '`table` is required.')
+    end
+
+    def job_status_polling_interval
+      @job_status_polling_interval ||= config[:job_status_polling_interval] || 5
+    end
+
+    def job_status_max_polling_time
+      @job_status_max_polling_time ||= config[:job_status_polling_time] || 3600
+    end
+
+    def dry_run?
+      @opts[:dry_run]
+    end
+
+    def head
+      dry_run? ? '(DRY-RUN) ' : '(EXECUTE) '
+    end
   end
 end

data/lib/bigquery_migration/version.rb

@@ -1,3 +1,3 @@
 class BigqueryMigration
-  VERSION = "0.2.2"
+  VERSION = "0.3.0"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: bigquery_migration
 version: !ruby/object:Gem::Version
-  version: 0.2.2
+  version: 0.3.0
 platform: ruby
 authors:
 - Naotoshi Seo
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2017-04-04 00:00:00.000000000 Z
+date: 2017-04-26 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: google-api-client
@@ -52,6 +52,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: inifile
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
@@ -157,6 +171,7 @@ files:
 - bigquery_migration.gemspec
 - bin/console
 - bin/setup
+- example/application_default.yml
 - example/copy_table.yml
 - example/example.yml
 - example/insert_select.yml
@@ -199,7 +214,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.5.2
+rubygems_version: 2.6.11
 signing_key: 
 specification_version: 4
 summary: Migrate BigQuery table schema