fluent-plugin-bigquery 0.0.2 → 0.0.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: caee66954ca996b43797e050825d5115dcd132f8
-  data.tar.gz: 040afb42fa0fe48f61a90d11e9ea5684d170974c
+  metadata.gz: 0802b40a0e2792d20ca2e92f20cfe398d92aa5db
+  data.tar.gz: a210aa8139c2d82467979adac99f6396096ef6d7
 SHA512:
-  metadata.gz: 15aed88d76491c467ff6e865381d1d78884d632f998c71569e53faadb1acf18d572e4459b55d4ff5a3e57121365cf1ec90c4c292726e492bcfdac6a52ab7f815
-  data.tar.gz: a5ed1a27a117a6f54a2931741c2ffdf37b856e54074af94e2740e70ed361fa21f4790a20e926d675019d1280574497303825562be0b75097200b5e58dd49646c
+  metadata.gz: 865a4f9cd3de0d912678768a5a29d3795b62382de2949ac7fc220446d2aaedb2c62616056ad2d44901c64c2782d45c3e2f2f1bb78ca73174394d1b2954291e5f
+  data.tar.gz: 1bde3427c107b3ac396e808f4856daff92f9a50666d07567dfe1b3cdd0fd50a0ca550b1573368b1913cde125dfa4d874ed6aa96862014fdb2bd22e950ce78bbf

data/README.md

@@ -9,14 +9,13 @@ Fluentd output plugin to load/insert data into Google BigQuery.
   * for data loading as batch jobs, for big amount of data
   * https://developers.google.com/bigquery/loading-data-into-bigquery
   
-Current version of this plugin supports Google API with Service Account Authentication, and does not support OAuth.
+Current version of this plugin supports Google API with Service Account Authentication, but does not support
+OAuth flow for installed applications.
 
 ## Configuration
 
 ### Streming inserts
 
-For service account authentication, generate service account private key file and email key, then upload private key file onto your server.
-
 Configure insert specifications with target table schema, with your credentials. This is minimum configurations:
 
 ```apache
@@ -25,6 +24,7 @@ Configure insert specifications with target table schema, with your credentials.
   
   method insert    # default
   
+  auth_method private_key   # default
   email xxxxxxxxxxxx-xxxxxxxxxxxxxxxxxxxxxx@developer.gserviceaccount.com
   private_key_path /home/username/.keys/00000000000000000000000000000000-privatekey.p12
   # private_key_passphrase notasecret # default
@@ -58,6 +58,7 @@ For high rate inserts over streaming inserts, you should specify flush intervals
   
   num_threads 16
   
+  auth_method private_key   # default
   email xxxxxxxxxxxx-xxxxxxxxxxxxxxxxxxxxxx@developer.gserviceaccount.com
   private_key_path /home/username/.keys/00000000000000000000000000000000-privatekey.p12
   # private_key_passphrase notasecret # default
@@ -100,6 +101,41 @@ Important options for high rate events are:
     * `1` is lowest value, without patches on Fluentd v0.10.41 or earlier
     * see `patches` below
 
+### Authentication
+
+There are two methods supported to fetch access token for the service account.
+1. Public-Private key pair
+2. Predefined access token (Compute Engine only)
+
+The examples above use the first one.  You first need to create a service account (client ID),
+download its private key and deploy the key with fluentd.
+
+On the other hand, you don't need to explicitly create a service account for fluentd when you
+run fluentd in Google Compute Engine.  In this second authentication method, you need to
+add the API scope "https://www.googleapis.com/auth/bigquery" to the scope list of your
+Compute Engine instance, then you can configure fluentd like this.
+
+```apache
+<match dummy>
+  type bigquery
+  
+  auth_method compute_engine
+  
+  project yourproject_id
+  dataset yourdataset_id
+  table   tablename
+  
+  time_format %s
+  time_field  time
+  
+  field_integer time,status,bytes
+  field_string  rhost,vhost,path,method,protocol,agent,referer
+  field_float   requestime
+  field_boolean bot_access,loginsession
+</match>
+```
+
+
 ### patches
 
 This plugin depends on `fluent-plugin-buffer-lightening`, and it includes monkey patch module for BufferedOutput plugin, to realize high rate and low latency flushing. With this patch, sub 1 second flushing available.

data/fluent-plugin-bigquery.gemspec

@@ -19,7 +19,7 @@ Gem::Specification.new do |spec|
   spec.require_paths = ["lib"]
 
   spec.add_development_dependency "rake"
-  spec.add_runtime_dependency "google-api-client", "~> 0.6.4"
+  spec.add_runtime_dependency "google-api-client", "~> 0.7.1"
   spec.add_runtime_dependency "fluentd"
   spec.add_runtime_dependency "fluent-mixin-plaintextformatter", '>= 0.2.1'
   spec.add_runtime_dependency "fluent-mixin-config-placeholders", ">= 0.2.0"

data/lib/fluent/plugin/bigquery/version.rb

@@ -1,6 +1,6 @@
 module Fluent
   module BigQueryPlugin
-    VERSION = "0.0.2"
+    VERSION = "0.0.3"
   end
 end
 

data/lib/fluent/plugin/out_bigquery.rb

@@ -39,9 +39,14 @@ module Fluent
     # config_param :client_id, :string
     # config_param :client_secret, :string
 
+    # Available methods are:
+    # * private_key -- Use service account credential
+    # * compute_engine -- Use access token available in instances of ComputeEngine
+    config_param :auth_method, :string, :default => 'private_key'
+
     ### Service Account credential
-    config_param :email, :string
-    config_param :private_key_path, :string
+    config_param :email, :string, :default => nil
+    config_param :private_key_path, :string, :default => nil
     config_param :private_key_passphrase, :string, :default => 'notasecret'
 
     # see as simple reference
@@ -115,11 +120,23 @@ module Fluent
       require 'google/api_client'
       require 'google/api_client/client_secrets'
       require 'google/api_client/auth/installed_app'
+      require 'google/api_client/auth/compute_service_account'
     end
 
     def configure(conf)
       super
 
+      case @auth_method
+      when 'private_key'
+        if !@email || !@private_key_path
+          raise Fluent::ConfigError, "'email' and 'private_key_path' must be specified if auth_method == 'private_key'"
+        end
+      when 'compute_engine'
+        # Do nothing
+      else
+        raise Fluent::ConfigError, "unrecognized 'auth_method': #{@auth_method}"
+      end
+
       if (!@table && !@tables) || (@table && @table)
         raise Fluent::ConfigError, "'table' or 'tables' must be specified, and both are invalid"
       end
@@ -180,14 +197,26 @@ module Fluent
         :application_version => Fluent::BigQueryPlugin::VERSION
       )
 
-      key = Google::APIClient::PKCS12.load_key( @private_key_path, @private_key_passphrase )
-      asserter = Google::APIClient::JWTAsserter.new(
-        @email,
-        "https://www.googleapis.com/auth/bigquery",
-        key
-      )
-      # refresh_auth
-      client.authorization = asserter.authorize
+      case @auth_method
+      when 'private_key'
+        key = Google::APIClient::PKCS12.load_key( @private_key_path, @private_key_passphrase )
+        asserter = Google::APIClient::JWTAsserter.new(
+          @email,
+          "https://www.googleapis.com/auth/bigquery",
+          key
+        )
+        # refresh_auth
+        client.authorization = asserter.authorize
+
+      when 'compute_engine'
+        auth = Google::APIClient::ComputeServiceAccount.new
+        auth.fetch_access_token!
+        client.authorization = auth
+
+      else
+        raise ConfigError, "Unknown auth method: #{@auth_method}"
+      end
+
       @cached_client_expiration = Time.now + 1800
       @cached_client = client
     end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-bigquery
 version: !ruby/object:Gem::Version
-  version: 0.0.2
+  version: 0.0.3
 platform: ruby
 authors:
 - TAGOMORI Satoshi
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-01-20 00:00:00.000000000 Z
+date: 2014-02-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -30,14 +30,14 @@ dependencies:
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 0.6.4
+        version: 0.7.1
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 0.6.4
+        version: 0.7.1
 - !ruby/object:Gem::Dependency
   name: fluentd
   requirement: !ruby/object:Gem::Requirement