fluent-plugin-bigquery 0.2.11 → 0.2.12

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: e93c58d0a0c1da28a59026a81f9d060ec3b0e532
-  data.tar.gz: ff810e3c6177a362a28a14cd92ea3091294212d4
+  metadata.gz: 24d39b32bcb8f6028618041edda11e1ffd3f6d16
+  data.tar.gz: a238724b34f64d36f7f319b0d6fc43f578b0dffc
 SHA512:
-  metadata.gz: 86e853fdad235e8ab2b014e100b4e1f355ff9e11da27605e30d66aa9676de3af888f8ab0499445b42df60c884114d5d380c4fa79f6b92ee226063554c4f1344c
-  data.tar.gz: a8e3d37c866073bfd54d25d15482387bb3454191ab5f6a41718c4a7b324d838e275b3dc2a8ce33eb213f57ee0d9a21b66cd878f5b3aec7f555be179238c4dedd
+  metadata.gz: 46e8ffbe6007cd2d855114671121285b4fb89b9fab378b59fcc37a5c20bd9f6587a5f8ab96a4f684cf9464d6d9da5b42b7a94f9d74e2cd6fd6769cf9c5f2f5df
+  data.tar.gz: 1abae1988128f8b2349aa898f1afb5f847e894c2ee88bd34126cb5981ffa0af5dd3fcf74c2e1f27fdc3c1d45814b2b000b932100f94e795ed27e8ab90e9b7894

data/README.md

@@ -110,15 +110,57 @@ section in the Google BigQuery document.
 
 There are two methods supported to fetch access token for the service account.
 
-1. Public-Private key pair
-2. Predefined access token (Compute Engine only)
+1. Public-Private key pair of GCP(Google Cloud Platform)'s service account
+2. JSON key of GCP(Google Cloud Platform)'s service account
+3. Predefined access token (Compute Engine only)
+4. Google application default credentials (http://goo.gl/IUuyuX)
 
-The examples above use the first one.  You first need to create a service account (client ID),
+#### Public-Private key pair of GCP's service account
+
+The examples above use the first one. You first need to create a service account (client ID),
 download its private key and deploy the key with fluentd.
 
-On the other hand, you don't need to explicitly create a service account for fluentd when you
-run fluentd in Google Compute Engine.  In this second authentication method, you need to
-add the API scope "https://www.googleapis.com/auth/bigquery" to the scope list of your
+#### JSON key of GCP(Google Cloud Platform)'s service account
+
+You first need to create a service account (client ID),
+download its JSON key and deploy the key with fluentd.
+
+```apache
+<match dummy>
+  type bigquery
+  
+  auth_method json_key
+  json_key /home/username/.keys/00000000000000000000000000000000-jsonkey.json
+  
+  project yourproject_id
+  dataset yourdataset_id
+  table   tablename
+  ...
+</match>
+```
+
+You can also provide `json_key` as embedded JSON string like this.
+You need to only include `private_key` and `client_email` key from JSON key file.
+
+```apache
+<match dummy>
+  type bigquery
+   
+  auth_method json_key
+  json_key {"private_key": "-----BEGIN PRIVATE KEY-----\n...", "client_email": "xxx@developer.gserviceaccount.com"}
+  
+  project yourproject_id
+  dataset yourdataset_id
+  table   tablename
+  ...
+</match>
+```
+
+#### Predefined access token (Compute Engine only)
+
+When you run fluentd on Googlce Compute Engine instance,
+you don't need to explicitly create a service account for fluentd.
+In this authentication method, you need to add the API scope "https://www.googleapis.com/auth/bigquery" to the scope list of your
 Compute Engine instance, then you can configure fluentd like this.
 
 ```apache
@@ -141,6 +183,19 @@ Compute Engine instance, then you can configure fluentd like this.
 </match>
 ```
 
+#### Application default credentials
+
+The Application Default Credentials provide a simple way to get authorization credentials for use in calling Google APIs, which are described in detail at http://goo.gl/IUuyuX.
+
+In this authentication method, the credentials returned are determined by the environment the code is running in. Conditions are checked in the following order:credentials are get from following order.
+
+1. The environment variable `GOOGLE_APPLICATION_CREDENTIALS` is checked. If this variable is specified it should point to a JSON key file that defines the credentials.
+2. The environment variable `GOOGLE_PRIVATE_KEY` and `GOOGLE_CLIENT_EMAIL` are checked. If this variables are specified `GOOGLE_PRIVATE_KEY` should point to `private_key`, `GOOGLE_CLIENT_EMAIL` should point to `client_email` in a JSON key.
+3. Well known path is checked. If file is exists, the file used as a JSON key file. This path is `$HOME/.config/gcloud/application_default_credentials.json`.
+4. System default path is checked. If file is exists, the file used as a JSON key file. This path is `/etc/google/auth/application_default_credentials.json`.
+5. If you are running in Google Compute Engine production, the built-in service account associated with the virtual machine instance will be used.
+6. If none of these conditions is true, an error will occur.
+
 ### Table id formatting
 
 `table` and `tables` options accept [Time#strftime](http://ruby-doc.org/core-1.9.3/Time.html#method-i-strftime)
@@ -282,9 +337,9 @@ You can set `insert_id_field` option to specify the field to use as `insertId` p
 ```apache
 <match dummy>
   type bigquery
-
+  
   ...
-
+  
   insert_id_field uuid
   field_string uuid
 </match>

data/fluent-plugin-bigquery.gemspec

@@ -24,6 +24,7 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "test-unit-rr", "~> 1.0.3"
 
   spec.add_runtime_dependency "google-api-client", "~> 0.8.0"
+  spec.add_runtime_dependency "googleauth"
   spec.add_runtime_dependency "fluentd"
   spec.add_runtime_dependency "fluent-mixin-plaintextformatter", '>= 0.2.1'
   spec.add_runtime_dependency "fluent-mixin-config-placeholders", ">= 0.3.0"

data/lib/fluent/plugin/bigquery/version.rb

@@ -1,6 +1,6 @@
 module Fluent
   module BigQueryPlugin
-    VERSION = "0.2.11"
+    VERSION = "0.2.12"
   end
 end
 

data/lib/fluent/plugin/out_bigquery.rb

@@ -38,14 +38,17 @@ module Fluent
     # config_param :client_secret, :string
 
     # Available methods are:
-    # * private_key -- Use service account credential
+    # * private_key -- Use service account credential from pkcs12 private key file
     # * compute_engine -- Use access token available in instances of ComputeEngine
+    # * private_json_key -- Use service account credential from JSON key
+    # * application_default -- Use application default credential
     config_param :auth_method, :string, default: 'private_key'
 
     ### Service Account credential
     config_param :email, :string, default: nil
     config_param :private_key_path, :string, default: nil
     config_param :private_key_passphrase, :string, default: 'notasecret', secret: true
+    config_param :json_key, default: nil
 
     # see as simple reference
     #   https://github.com/abronte/BigQuery/blob/master/lib/bigquery.rb
@@ -128,9 +131,10 @@ module Fluent
       super
       require 'json'
       require 'google/api_client'
-      require 'google/api_client/client_secrets'
-      require 'google/api_client/auth/installed_app'
-      require 'google/api_client/auth/compute_service_account'
+      require 'googleauth'
+
+      # MEMO: signet-0.6.1 depend on Farady.default_connection
+      Faraday.default_connection.options.timeout = 60
     end
 
     # Define `log` method for v0.10.42 or earlier
@@ -148,6 +152,12 @@ module Fluent
         end
       when 'compute_engine'
         # Do nothing
+      when 'json_key'
+        unless @json_key
+          raise Fluent::ConfigError, "'json_key' must be specified if auth_method == 'json_key'"
+        end
+      when 'application_default'
+        # Do nothing
       else
         raise Fluent::ConfigError, "unrecognized 'auth_method': #{@auth_method}"
       end
@@ -227,26 +237,41 @@ module Fluent
         application_version: Fluent::BigQueryPlugin::VERSION
       )
 
+      scope = "https://www.googleapis.com/auth/bigquery"
+
       case @auth_method
       when 'private_key'
-        key = Google::APIClient::PKCS12.load_key( @private_key_path, @private_key_passphrase )
-        asserter = Google::APIClient::JWTAsserter.new(
-          @email,
-          "https://www.googleapis.com/auth/bigquery",
-          key
-        )
-        # refresh_auth
-        client.authorization = asserter.authorize
+        key = Google::APIClient::KeyUtils.load_from_pkcs12(@private_key_path, @private_key_passphrase)
+        auth = Signet::OAuth2::Client.new(
+                token_credential_uri: "https://accounts.google.com/o/oauth2/token",
+                audience: "https://accounts.google.com/o/oauth2/token",
+                scope: scope,
+                issuer: @email,
+                signing_key: key)
 
       when 'compute_engine'
-        auth = Google::APIClient::ComputeServiceAccount.new
-        auth.fetch_access_token!
-        client.authorization = auth
+        auth = Google::Auth::GCECredentials.new
+
+      when 'json_key'
+        if File.exist?(@json_key)
+          auth = File.open(@json_key) do |f|
+            Google::Auth::ServiceAccountCredentials.new(json_key_io: f, scope: scope)
+          end
+        else
+          key = StringIO.new(@json_key)
+          auth = Google::Auth::ServiceAccountCredentials.new(json_key_io: key, scope: scope)
+        end
+
+      when 'application_default'
+        auth = Google::Auth.get_application_default([scope])
 
       else
         raise ConfigError, "Unknown auth method: #{@auth_method}"
       end
 
+      auth.fetch_access_token!
+      client.authorization = auth
+
       @cached_client_expiration = Time.now + 1800
       @cached_client = client
     end
@@ -349,7 +374,7 @@ module Fluent
         if @replace_record_key
           record = replace_record_key(record)
         end
-        
+
         row = @fields.format(@add_time_field.call(record, time))
         unless row.empty?
           row = {"json" => row}
@@ -442,7 +467,7 @@ module Fluent
         ### https://developers.google.com/bigquery/docs/tables
         # Each field has the following properties:
         #
-        # name - The name must contain only letters (a-z, A-Z), numbers (0-9), or underscores (_), 
+        # name - The name must contain only letters (a-z, A-Z), numbers (0-9), or underscores (_),
         #        and must start with a letter or underscore. The maximum length is 128 characters.
         #        https://cloud.google.com/bigquery/docs/reference/v2/tables#schema.fields.name
         unless name =~ /^[_A-Za-z][_A-Za-z0-9]{,127}$/
@@ -468,7 +493,7 @@ module Fluent
       end
 
       def format_one(value)
-        raise NotImplementedError, "Must implement in a subclass" 
+        raise NotImplementedError, "Must implement in a subclass"
       end
 
       def to_h

data/test/plugin/test_out_bigquery.rb

@@ -1,5 +1,6 @@
 require 'helper'
 require 'google/api_client'
+require 'googleauth'
 require 'fluent/plugin/buf_memory'
 
 class BigQueryOutputTest < Test::Unit::TestCase
@@ -60,14 +61,20 @@ class BigQueryOutputTest < Test::Unit::TestCase
     }
   end
 
-  def test_configure_auth
+  def test_configure_auth_private_key
     key = stub!
-    mock(Google::APIClient::PKCS12).load_key('/path/to/key', 'notasecret') { key }
+    mock(Google::APIClient::KeyUtils).load_from_pkcs12('/path/to/key', 'notasecret') { key }
     authorization = Object.new
-    asserter = mock!.authorize { authorization }
-    mock(Google::APIClient::JWTAsserter).new('foo@bar.example', API_SCOPE, key) { asserter }
-
-    mock.proxy(Google::APIClient).new.with_any_args { 
+    mock(authorization).fetch_access_token!
+    stub(Signet::OAuth2::Client).new
+    mock(Signet::OAuth2::Client).new(
+      token_credential_uri: "https://accounts.google.com/o/oauth2/token",
+      audience: "https://accounts.google.com/o/oauth2/token",
+      scope: API_SCOPE,
+      issuer: 'foo@bar.example',
+      signing_key: key) { authorization }
+
+    mock.proxy(Google::APIClient).new.with_any_args {
       mock!.__send__(:authorization=, authorization) {}
     }
 
@@ -75,6 +82,88 @@ class BigQueryOutputTest < Test::Unit::TestCase
     driver.instance.client()
   end
 
+  def test_configure_auth_compute_engine
+    authorization = Object.new
+    mock(authorization).fetch_access_token!
+    mock(Google::Auth::GCECredentials).new { authorization }
+
+    mock.proxy(Google::APIClient).new.with_any_args {
+      mock!.__send__(:authorization=, authorization) {}
+    }
+
+    driver = create_driver(%[
+      table foo
+      auth_method compute_engine
+      project yourproject_id
+      dataset yourdataset_id
+      field_integer time,status,bytes
+    ])
+    driver.instance.client()
+  end
+
+  def test_configure_auth_json_key_as_file
+    json_key_path = 'test/plugin/testdata/json_key.json'
+    authorization = Object.new
+    mock(authorization).fetch_access_token!
+    mock(Google::Auth::ServiceAccountCredentials).new(json_key_io: File.open(json_key_path), scope: API_SCOPE) { authorization }
+
+    mock.proxy(Google::APIClient).new.with_any_args {
+      mock!.__send__(:authorization=, authorization) {}
+    }
+
+    driver = create_driver(%[
+      table foo
+      auth_method json_key
+      json_key #{json_key_path}
+      project yourproject_id
+      dataset yourdataset_id
+      field_integer time,status,bytes
+    ])
+    driver.instance.client()
+  end
+
+  def test_configure_auth_json_key_as_string
+    json_key = '{"private_key": "X", "client_email": "xxx@developer.gserviceaccount.com"}'
+    json_key_io = StringIO.new(json_key)
+    mock(StringIO).new(json_key) { json_key_io }
+    authorization = Object.new
+    mock(authorization).fetch_access_token!
+    mock(Google::Auth::ServiceAccountCredentials).new(json_key_io: json_key_io, scope: API_SCOPE) { authorization }
+
+    mock.proxy(Google::APIClient).new.with_any_args {
+      mock!.__send__(:authorization=, authorization) {}
+    }
+
+    driver = create_driver(%[
+      table foo
+      auth_method json_key
+      json_key #{json_key}
+      project yourproject_id
+      dataset yourdataset_id
+      field_integer time,status,bytes
+    ])
+    driver.instance.client()
+  end
+
+  def test_configure_auth_application_default
+    authorization = Object.new
+    mock(authorization).fetch_access_token!
+    mock(Google::Auth).get_application_default([API_SCOPE]) { authorization }
+
+    mock.proxy(Google::APIClient).new.with_any_args {
+      mock!.__send__(:authorization=, authorization) {}
+    }
+
+    driver = create_driver(%[
+      table foo
+      auth_method application_default
+      project yourproject_id
+      dataset yourdataset_id
+      field_integer time,status,bytes
+    ])
+    driver.instance.client()
+  end
+
   def test_configure_fieldname_stripped
     driver = create_driver(%[
       table foo
@@ -86,10 +175,10 @@ class BigQueryOutputTest < Test::Unit::TestCase
       time_format %s
       time_field  time
 
-      field_integer time  , status , bytes 
+      field_integer time  , status , bytes
       field_string  _log_name, vhost, path, method, protocol, agent, referer, remote.host, remote.ip, remote.user
-      field_float   requesttime 
-      field_boolean bot_access , loginsession 
+      field_float   requesttime
+      field_boolean bot_access , loginsession
     ])
     fields = driver.instance.instance_eval{ @fields }
 
@@ -483,7 +572,7 @@ class BigQueryOutputTest < Test::Unit::TestCase
     assert fields["tty"]
     assert_equal :string, fields["tty"].type
     assert_equal :nullable, fields["tty"].mode
-    
+
     assert fields["pwd"]
     assert_equal :string, fields["pwd"].type
     assert_equal :required, fields["pwd"].mode
@@ -560,7 +649,7 @@ class BigQueryOutputTest < Test::Unit::TestCase
     assert fields["tty"]
     assert_equal :string, fields["tty"].type
     assert_equal :nullable, fields["tty"].mode
-    
+
     assert fields["pwd"]
     assert_equal :string, fields["pwd"].type
     assert_equal :required, fields["pwd"].mode

data/test/plugin/testdata/json_key.json

@@ -0,0 +1,7 @@
+{
+  "private_key_id": "1",
+  "private_key": "X",
+  "client_email": "xxx@developer.gserviceaccount.com",
+  "client_id": "xxx.apps.googleusercontent.com",
+  "type": "service_account"
+}

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-bigquery
 version: !ruby/object:Gem::Version
-  version: 0.2.11
+  version: 0.2.12
 platform: ruby
 authors:
 - Naoya Ito
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-08-24 00:00:00.000000000 Z
+date: 2015-10-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -80,6 +80,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: 0.8.0
+- !ruby/object:Gem::Dependency
+  name: googleauth
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: fluentd
   requirement: !ruby/object:Gem::Requirement
@@ -171,6 +185,7 @@ files:
 - test/helper.rb
 - test/plugin/test_out_bigquery.rb
 - test/plugin/testdata/apache.schema
+- test/plugin/testdata/json_key.json
 - test/plugin/testdata/sudo.schema
 - test/test_load_request_body_wrapper.rb
 homepage: https://github.com/kaizenplatform/fluent-plugin-bigquery
@@ -201,5 +216,6 @@ test_files:
 - test/helper.rb
 - test/plugin/test_out_bigquery.rb
 - test/plugin/testdata/apache.schema
+- test/plugin/testdata/json_key.json
 - test/plugin/testdata/sudo.schema
 - test/test_load_request_body_wrapper.rb