fluent-plugin-bigquery 0.2.16 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 6283655314f920c8d3f1bab8f387d96c6fe79da0
-  data.tar.gz: f1016e03203cf12c4c26ad62f1c3a05926423fa7
+  metadata.gz: 77b839947c8f721f341499a0be5c9f21552833ad
+  data.tar.gz: 3efa172577cb54e19290f0df245ecbb677994869
 SHA512:
-  metadata.gz: 15e484f5df810cd5736711bd70df5a9e34950e10c77118a3b6097fba6f9c1efd9641ac515df547a15b2f9dac653deb3d5b2fa665541a47bf43dba750754d584e
-  data.tar.gz: 1bbcea1f4ec490c69028eca66032b6dca734231fedf431951618b1d5ad08a354395f357e31a028f3e60531ec15e831bbbb0d4f9a70bd48082a57562885564023
+  metadata.gz: f0dbcd7b1cd8f4462657006f9a51338e0327f0b61a172ae0d39860a4c88bae4af0ba8f299cc9657a8ac3eb44acf53f5bc1161d023a06b5ab6d133d4ab72aeba2
+  data.tar.gz: aabd97536deeeb1a6b3f55e0c71f28ddbcfbf99ced84c7d299e881409dab907a34a943c8fa69df0a3373d6c6e1eb5a83fcde157ea6c601b90d84ad7c93b816b4

data/README.md

@@ -2,6 +2,8 @@
 
 [Fluentd](http://fluentd.org) output plugin to load/insert data into Google BigQuery.
 
+- **Plugin type**: TimeSlicedOutput
+
 * insert data over streaming inserts
   * for continuous real-time insertions
   * https://developers.google.com/bigquery/streaming-data-into-bigquery#usecases
@@ -14,6 +16,59 @@ OAuth flow for installed applications.
 
 ## Configuration
 
+### Options
+
+| name                                   | type          | required?                   | default                                                | description                                                                                                          |
+| :------------------------------------- | :------------ | :-----------                | :-------------------------                             | :-----------------------                                                                                             |
+| method                                 | string        | no                          | insert                                                 | `insert` (Streaming Insert) or `load` (load job)                                                                     |
+| buffer_type                            | string        | no                          | lightening (insert) or file (load)                     |                                                                                                                      |
+| buffer_chunk_limit                     | integer       | no                          | 1MB (insert) or 1GB (load)                             |                                                                                                                      |
+| buffer_queue_limit                     | integer       | no                          | 1024 (insert) or 32 (load)                             |                                                                                                                      |
+| buffer_chunk_records_limit             | integer       | no                          | 500                                                    |                                                                                                                      |
+| flush_interval                         | float         | no                          | 0.25 (*insert) or default of time sliced output (load) |                                                                                                                      |
+| try_flush_interval                     | float         | no                          | 0.05 (*insert) or default of time sliced output (load) |                                                                                                                      |
+| auth_method                            | enum          | yes                         | private_key                                            | `private_key` or `json_key` or `compute_engine` or `application_default`                                             |
+| email                                  | string        | yes (private_key)           | nil                                                    | GCP Service Account Email                                                                                            |
+| private_key_path                       | string        | yes (private_key)           | nil                                                    | GCP Private Key file path                                                                                            |
+| private_key_passphrase                 | string        | yes (private_key)           | nil                                                    | GCP Private Key Passphrase                                                                                           |
+| json_key                               | string        | yes (json_key)              | nil                                                    | GCP JSON Key file path or JSON Key string                                                                            |
+| project                                | string        | yes                         | nil                                                    |                                                                                                                      |
+| table                                  | string        | yes (either `tables`)       | nil                                                    |                                                                                                                      |
+| tables                                 | string        | yes (either `table`)        | nil                                                    | can set multi table names splitted by `,`                                                                            |
+| template_suffix                        | string        | no                          | nil                                                    | can use `%{time_slice}` placeholder replaced by `time_slice_format`                                                  |
+| auto_create_table                      | bool          | no                          | false                                                  | If true, creates table automatically                                                                                 |
+| skip_invalid_rows                      | bool          | no                          | false                                                  | Only `insert` method.                                                                                                |
+| max_bad_records                        | integer       | no                          | 0                                                      | Only `load` method. If the number of bad records exceeds this value, an invalid error is returned in the job result. |
+| ignore_unknown_values                  | bool          | no                          | false                                                  | Accept rows that contain values that do not match the schema. The unknown values are ignored.                        |
+| schema_path                            | string        | yes (either `fetch_schema`) | nil                                                    | Schema Definition file path. It is formatted by JSON.                                                                |
+| fetch_schema                           | bool          | yes (either `schema_path`)  | false                                                  | If true, fetch table schema definition from Bigquery table automatically.                                            |
+| fetch_schema_table                     | string        | no                          | nil                                                    | If set, fetch table schema definition from this table, If fetch_schema is false, this param is ignored               |
+| schema_cache_expire                    | integer       | no                          | 600                                                    | Value is second. If current time is after expiration interval, re-fetch table schema definition.                     |
+| field_string                           | string        | no                          | nil                                                    | see examples.                                                                                                        |
+| field_integer                          | string        | no                          | nil                                                    | see examples.                                                                                                        |
+| field_float                            | string        | no                          | nil                                                    | see examples.                                                                                                        |
+| field_boolean                          | string        | no                          | nil                                                    | see examples.                                                                                                        |
+| field_timestamp                        | string        | no                          | nil                                                    | see examples.                                                                                                        |
+| time_field                             | string        | no                          | nil                                                    | If this param is set, plugin set formatted time string to this field.                                                |
+| time_format                            | string        | no                          | nil                                                    | ex. `%s`, `%Y/%m%d %H:%M:%S`                                                                                         |
+| replace_record_key                     | bool          | no                          | false                                                  | see examples.                                                                                                        |
+| replace_record_key_regexp{1-10}        | string        | no                          | nil                                                    | see examples.                                                                                                        |
+| convert_hash_to_json                   | bool          | no                          | false                                                  | If true, converts Hash value of record to JSON String.                                                               |
+| insert_id_field                        | string        | no                          | nil                                                    | Use key as `insert_id` of Streaming Insert API parameter.                                                            |
+| request_timeout_sec                    | integer       | no                          | nil                                                    | Bigquery API response timeout                                                                                        |
+| request_open_timeout_sec               | integer       | no                          | 60                                                     | Bigquery API connection, and request timeout. If you send big data to Bigquery, set large value.                     |
+
+### Standard Options
+
+| name                                   | type          | required?    | default                    | description              |
+| :------------------------------------- | :------------ | :----------- | :------------------------- | :----------------------- |
+| localtime                              | bool          | no           | nil                        | Use localtime            |
+| utc                                    | bool          | no           | nil                        | Use utc                  |
+
+And see http://docs.fluentd.org/articles/output-plugin-overview#time-sliced-output-parameters
+
+## Examples
+
 ### Streaming inserts
 
 Configure insert specifications with target table schema, with your credentials. This is minimum configurations:
@@ -139,7 +194,7 @@ __CAUTION: `flush_interval` default is still `0.25` even if `method` is `load` o
 
 ### Authentication
 
-There are two methods supported to fetch access token for the service account.
+There are four methods supported to fetch access token for the service account.
 
 1. Public-Private key pair of GCP(Google Cloud Platform)'s service account
 2. JSON key of GCP(Google Cloud Platform)'s service account
@@ -301,6 +356,24 @@ For example value of `subdomain` attribute is `"bq.fluent"`, table id will be li
 - any type of attribute is allowed because stringified value will be used as replacement.
 - acceptable characters are alphabets, digits and `_`. All other characters will be removed.
 
+### Date partitioned table support
+this plugin can insert (load) into date partitioned table.
+
+Use `%{time_slice}`.
+
+```apache
+<match dummy>
+  @type bigquery
+
+  ...
+  time_slice_format %Y%m%d
+  table   accesslog$%{time_slice}
+  ...
+</match>
+```
+
+But, Dynamic table creating doesn't support date partitioned table yet.
+
 ### Dynamic table creating
 
 When `auto_create_table` is set to `true`, try to create the table using BigQuery API when insertion failed with code=404 "Not Found: Table ...".
@@ -398,6 +471,7 @@ The third method is to set `fetch_schema` to `true` to enable fetch a schema usi
   time_field  time
   
   fetch_schema true
+  # fetch_schema_table other_table # if you want to fetch schema from other table
   field_integer time
 </match>
 ```
@@ -425,8 +499,6 @@ You can set `insert_id_field` option to specify the field to use as `insertId` p
 
 ## TODO
 
-* support Load API
-  * with automatically configured flush/buffer options
 * support optional data fields
 * support NULLABLE/REQUIRED/REPEATED field options in field list style of configuration
 * OAuth installed application credentials support
@@ -438,3 +510,4 @@ You can set `insert_id_field` option to specify the field to use as `insertId` p
 
 * @tagomoris: First author, original version
 * KAIZEN platform Inc.: Maintener, Since 2014.08.19
+* @joker1007

data/Rakefile

@@ -6,6 +6,7 @@ Rake::TestTask.new(:test) do |test|
   test.libs << 'lib' << 'test'
   test.pattern = 'test/**/test_*.rb'
   test.verbose = true
+  test.warning = false
 end
 
 task :default => :test

data/fluent-plugin-bigquery.gemspec

@@ -24,13 +24,11 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "test-unit-rr", "~> 1.0.3"
 
   spec.add_runtime_dependency "google-api-client", "~> 0.9.3"
-  spec.add_runtime_dependency "googleauth", ">=  0.5.0"
+  spec.add_runtime_dependency "googleauth", ">= 0.5.0"
   spec.add_runtime_dependency "multi_json"
-  spec.add_runtime_dependency "activesupport", ">= 3.2"
-  spec.add_runtime_dependency "fluentd"
+  spec.add_runtime_dependency "activesupport", ">= 3.2", "< 5"
+  spec.add_runtime_dependency "fluentd", "~> 0.12.0"
   spec.add_runtime_dependency "fluent-mixin-plaintextformatter", '>= 0.2.1'
   spec.add_runtime_dependency "fluent-mixin-config-placeholders", ">= 0.3.0"
   spec.add_runtime_dependency "fluent-plugin-buffer-lightening", ">= 0.0.2"
-
-  spec.add_development_dependency "fluent-plugin-dummydata-producer"
 end

data/lib/fluent/plugin/bigquery/schema.rb

@@ -0,0 +1,221 @@
+module Fluent
+  module BigQuery
+    class FieldSchema
+      def initialize(name, mode = :nullable)
+        unless [:nullable, :required, :repeated].include?(mode)
+          raise ConfigError, "Unrecognized mode for #{name}: #{mode}"
+        end
+        ### https://developers.google.com/bigquery/docs/tables
+        # Each field has the following properties:
+        #
+        # name - The name must contain only letters (a-z, A-Z), numbers (0-9), or underscores (_),
+        #        and must start with a letter or underscore. The maximum length is 128 characters.
+        #        https://cloud.google.com/bigquery/docs/reference/v2/tables#schema.fields.name
+        unless name =~ /^[_A-Za-z][_A-Za-z0-9]{,127}$/
+          raise ConfigError, "invalid bigquery field name: '#{name}'"
+        end
+
+        @name = name
+        @mode = mode
+      end
+
+      attr_reader :name, :mode
+
+      def format(value)
+        case @mode
+        when :nullable
+          format_one(value) unless value.nil?
+        when :required
+          if value.nil?
+            log.warn "Required field #{name} cannot be null"
+            nil
+          else
+            format_one(value)
+          end
+        when :repeated
+          value.nil? ? [] : value.each_with_object([]) { |v, arr| arr << format_one(v) if v }
+        end
+      end
+
+      def format_one(value)
+        raise NotImplementedError, "Must implement in a subclass"
+      end
+
+      def to_h
+        {
+          :name => name,
+          :type => type.to_s.upcase,
+          :mode => mode.to_s.upcase,
+        }
+      end
+    end
+
+    class StringFieldSchema < FieldSchema
+      def type
+        :string
+      end
+
+      def format_one(value)
+        value.to_s
+      end
+    end
+
+    class IntegerFieldSchema < FieldSchema
+      def type
+        :integer
+      end
+
+      def format_one(value)
+        value.to_i
+      end
+    end
+
+    class FloatFieldSchema < FieldSchema
+      def type
+        :float
+      end
+
+      def format_one(value)
+        value.to_f
+      end
+    end
+
+    class BooleanFieldSchema < FieldSchema
+      def type
+        :boolean
+      end
+
+      def format_one(value)
+        !!value
+      end
+    end
+
+    class TimestampFieldSchema < FieldSchema
+      INTEGER_REGEXP = /\A-?[[:digit:]]+\z/.freeze
+      FLOAT_REGEXP = /\A-?[[:digit:]]+(\.[[:digit:]]+)\z/.freeze
+
+      def type
+        :timestamp
+      end
+
+      def format_one(value)
+        case value
+        when Time
+          value.strftime("%Y-%m-%d %H:%M:%S.%6L %:z")
+        when String
+          if value =~ INTEGER_REGEXP
+            value.to_i
+          elsif value =~ FLOAT_REGEXP
+            value.to_f
+          else
+            value
+          end
+        else
+          value
+        end
+      end
+    end
+
+    class RecordSchema < FieldSchema
+      FIELD_TYPES = {
+        string: StringFieldSchema,
+        integer: IntegerFieldSchema,
+        float: FloatFieldSchema,
+        boolean: BooleanFieldSchema,
+        timestamp: TimestampFieldSchema,
+        record: RecordSchema
+      }.freeze
+
+      def initialize(name, mode = :nullable)
+        super(name, mode)
+        @fields = {}
+      end
+
+      def type
+        :record
+      end
+
+      def [](name)
+        @fields[name]
+      end
+
+      def empty?
+        @fields.empty?
+      end
+
+      def to_a
+        @fields.map do |_, field_schema|
+          field_schema.to_h
+        end
+      end
+
+      def to_h
+        {
+          :name => name,
+          :type => type.to_s.upcase,
+          :mode => mode.to_s.upcase,
+          :fields => self.to_a,
+        }
+      end
+
+      def load_schema(schema, allow_overwrite=true)
+        schema.each do |field|
+          raise ConfigError, 'field must have type' unless field.key?('type')
+
+          name = field['name']
+          mode = (field['mode'] || 'nullable').downcase.to_sym
+
+          type = field['type'].downcase.to_sym
+          field_schema_class = FIELD_TYPES[type]
+          raise ConfigError, "Invalid field type: #{field['type']}" unless field_schema_class
+
+          next if @fields.key?(name) and !allow_overwrite
+
+          field_schema = field_schema_class.new(name, mode)
+          @fields[name] = field_schema
+          if type == :record
+            raise ConfigError, "record field must have fields" unless field.key?('fields')
+            field_schema.load_schema(field['fields'], allow_overwrite)
+          end
+        end
+      end
+
+      def register_field(name, type)
+        if @fields.key?(name) and @fields[name].type != :timestamp
+          raise ConfigError, "field #{name} is registered twice"
+        end
+        if name[/\./]
+          recordname = $`
+          fieldname = $'
+          register_record_field(recordname)
+          @fields[recordname].register_field(fieldname, type)
+        else
+          schema = FIELD_TYPES[type]
+          raise ConfigError, "[Bug] Invalid field type #{type}" unless schema
+          @fields[name] = schema.new(name)
+        end
+      end
+
+      def format_one(record)
+        out = {}
+        record.each do |key, value|
+          next if value.nil?
+          schema = @fields[key]
+          out[key] = schema ? schema.format(value) : value
+        end
+        out
+      end
+
+      private
+      def register_record_field(name)
+        if !@fields.key?(name)
+          @fields[name] = RecordSchema.new(name)
+        else
+          unless @fields[name].kind_of?(RecordSchema)
+            raise ConfigError, "field #{name} is required to be a record but already registered as #{@field[name]}"
+          end
+        end
+      end
+    end
+  end
+end

data/lib/fluent/plugin/bigquery/version.rb

@@ -1,6 +1,6 @@
 module Fluent
   module BigQueryPlugin
-    VERSION = "0.2.16"
+    VERSION = "0.3.0"
   end
 end
 

data/lib/fluent/plugin/bigquery/writer.rb

@@ -0,0 +1,289 @@
+module Fluent
+  module BigQuery
+    class Writer
+      RETRYABLE_ERROR_REASON = %w(backendError internalError rateLimitExceeded tableUnavailable).freeze
+
+      class Error < StandardError
+        attr_reader :origin
+
+        def initialize(message, origin = nil)
+          @origin = origin
+          super(message || origin.message)
+        end
+
+        def method_missing(name, *args)
+          if @origin
+            @origin.send(name, *args)
+          else
+            super
+          end
+        end
+
+        def reason
+          @origin && @origin.respond_to?(:reason) ? @origin.reason : nil
+        end
+
+        def status_code
+          @origin && @origin.respond_to?(:status_code) ? @origin.status_code : nil
+        end
+
+        def retryable?
+          false
+        end
+      end
+
+      class UnRetryableError < Error; end
+
+      class RetryableError < Error
+        def retryable?
+          true
+        end
+      end
+
+      def initialize(log, auth_method, auth_options = {})
+        @auth_method = auth_method
+        @scope = "https://www.googleapis.com/auth/bigquery"
+        @auth_options = auth_options
+        @log = log
+
+        @cached_client_expiration = Time.now + 1800
+      end
+
+      def client
+        return @client if @client && @cached_client_expiration > Time.now
+
+        client = Google::Apis::BigqueryV2::BigqueryService.new.tap do |cl|
+          cl.authorization = get_auth
+        end
+
+        @cached_client_expiration = Time.now + 1800
+        @client = client
+      end
+
+      def create_table(project, dataset, table_id, record_schema)
+        create_table_retry_limit = 3
+        create_table_retry_wait = 1
+        create_table_retry_count = 0
+
+        begin
+          client.insert_table(project, dataset, {
+            table_reference: {
+              table_id: table_id,
+            },
+            schema: {
+              fields: record_schema.to_a,
+            }
+          }, {})
+          log.debug "create table", project_id: project, dataset: dataset, table: table_id
+          @client = nil
+        rescue Google::Apis::ServerError, Google::Apis::ClientError, Google::Apis::AuthorizationError => e
+          @client = nil
+
+          message = e.message
+          if e.status_code == 409 && /Already Exists:/ =~ message
+            log.debug "already created table", project_id: project, dataset: dataset, table: table_id
+            # ignore 'Already Exists' error
+            return
+          end
+
+          reason = e.respond_to?(:reason) ? e.reason : nil
+          log.error "tables.insert API", project_id: project, dataset: dataset, table: table_id, code: e.status_code, message: message, reason: reason
+
+          if RETRYABLE_ERROR_REASON.include?(reason) && create_table_retry_count < create_table_retry_limit
+            sleep create_table_retry_wait
+            create_table_retry_wait *= 2
+            create_table_retry_count += 1
+            retry
+          else
+            raise UnRetryableError.new("failed to create table in bigquery", e)
+          end
+        end
+      end
+
+      def fetch_schema(project, dataset, table_id)
+        res = client.get_table(project, dataset, table_id)
+        schema = res.schema.fields.as_json
+        log.debug "Load schema from BigQuery: #{project}:#{dataset}.#{table_id} #{schema}"
+
+        schema
+      rescue Google::Apis::ServerError, Google::Apis::ClientError, Google::Apis::AuthorizationError => e
+        @client = nil
+        message = e.message
+        log.error "tables.get API", project_id: project, dataset: dataset, table: table_id, code: e.status_code, message: message
+        nil
+      end
+
+      def insert_rows(project, dataset, table_id, rows, skip_invalid_rows: false, ignore_unknown_values: false, template_suffix: nil, timeout_sec: nil, open_timeout_sec: 60)
+        body = {
+          rows: rows,
+          skip_invalid_rows: skip_invalid_rows,
+          ignore_unknown_values: ignore_unknown_values,
+        }
+        body.merge!(template_suffix: template_suffix) if template_suffix
+        res = client.insert_all_table_data(project, dataset, table_id, body, {
+          options: {timeout_sec: timeout_sec, open_timeout_sec: open_timeout_sec}
+        })
+        log.debug "insert rows", project_id: project, dataset: dataset, table: table_id, count: rows.size
+        log.warn "insert errors", insert_errors: res.insert_errors if res.insert_errors && !res.insert_errors.empty?
+      rescue Google::Apis::ServerError, Google::Apis::ClientError, Google::Apis::AuthorizationError => e
+        @client = nil
+
+        reason = e.respond_to?(:reason) ? e.reason : nil
+        log.error "tabledata.insertAll API", project_id: project, dataset: dataset, table: table_id, code: e.status_code, message: e.message, reason: reason
+
+        if RETRYABLE_ERROR_REASON.include?(reason)
+          raise RetryableError.new(nil, e)
+        else
+          raise UnRetryableError.new(nil, e)
+        end
+      end
+
+      def create_load_job(project, dataset, table_id, upload_source, job_id, fields, ignore_unknown_values: false, max_bad_records: 0, timeout_sec: nil, open_timeout_sec: 60)
+        configuration = {
+          configuration: {
+            load: {
+              destination_table: {
+                project_id: project,
+                dataset_id: dataset,
+                table_id: table_id,
+              },
+              schema: {
+                fields: fields.to_a,
+              },
+              write_disposition: "WRITE_APPEND",
+              source_format: "NEWLINE_DELIMITED_JSON",
+              ignore_unknown_values: ignore_unknown_values,
+              max_bad_records: max_bad_records,
+            }
+          }
+        }
+        configuration.merge!({job_reference: {project_id: project, job_id: job_id}}) if job_id
+
+        # If target table is already exist, omit schema configuration.
+        # Because schema changing is easier.
+        begin
+          if client.get_table(project, dataset, table_id)
+            configuration[:configuration][:load].delete(:schema)
+          end
+        rescue Google::Apis::ServerError, Google::Apis::ClientError, Google::Apis::AuthorizationError
+          raise UnRetryableError.new("Schema is empty") if fields.empty?
+        end
+
+        res = client.insert_job(
+          project,
+          configuration,
+          {
+            upload_source: upload_source,
+            content_type: "application/octet-stream",
+            options: {
+              timeout_sec: timeout_sec,
+              open_timeout_sec: open_timeout_sec,
+            }
+          }
+        )
+        wait_load_job(project, dataset, res.job_reference.job_id, table_id)
+      rescue Google::Apis::ServerError, Google::Apis::ClientError, Google::Apis::AuthorizationError => e
+        @client = nil
+
+        reason = e.respond_to?(:reason) ? e.reason : nil
+        log.error "job.load API", project_id: project, dataset: dataset, table: table_id, code: e.status_code, message: e.message, reason: reason
+
+        return wait_load_job(project, dataset, job_id, table_id, retryable: false) if job_id && e.status_code == 409 && e.message =~ /Job/ # duplicate load job
+
+        if RETRYABLE_ERROR_REASON.include?(reason) || e.is_a?(Google::Apis::ServerError)
+          raise RetryableError.new(nil, e)
+        else
+          raise UnRetryableError.new(nil, e)
+        end
+      end
+
+      def wait_load_job(project, dataset, job_id, table_id, retryable: true)
+        wait_interval = 10
+        _response = client.get_job(project, job_id)
+
+        until _response.status.state == "DONE"
+          log.debug "wait for load job finish", state: _response.status.state, job_id: _response.job_reference.job_id
+          sleep wait_interval
+          _response = client.get_job(project, _response.job_reference.job_id)
+        end
+
+        errors = _response.status.errors
+        if errors
+          errors.each do |e|
+            log.error "job.insert API (rows)", job_id: job_id, project_id: project, dataset: dataset, table: table_id, message: e.message, reason: e.reason
+          end
+        end
+
+        error_result = _response.status.error_result
+        if error_result
+          log.error "job.insert API (result)", job_id: job_id, project_id: project, dataset: dataset, table: table_id, message: error_result.message, reason: error_result.reason
+          if retryable && RETRYABLE_ERROR_REASON.include?(error_result.reason)
+            raise RetryableError.new("failed to load into bigquery, retry")
+          else
+            raise UnRetryableError.new("failed to load into bigquery, and cannot retry")
+          end
+        end
+
+        log.debug "finish load job", state: _response.status.state
+      end
+
+      private
+
+      def log
+        @log
+      end
+
+      def get_auth
+        case @auth_method
+        when :private_key
+          get_auth_from_private_key
+        when :compute_engine
+          get_auth_from_compute_engine
+        when :json_key
+          get_auth_from_json_key
+        when :application_default
+          get_auth_from_application_default
+        else
+          raise ConfigError, "Unknown auth method: #{@auth_method}"
+        end
+      end
+
+      def get_auth_from_private_key
+        require 'google/api_client/auth/key_utils'
+        private_key_path = @auth_options[:private_key_path]
+        private_key_passphrase = @auth_options[:private_key_passphrase]
+        email = @auth_options[:email]
+
+        key = Google::APIClient::KeyUtils.load_from_pkcs12(private_key_path, private_key_passphrase)
+        Signet::OAuth2::Client.new(
+          token_credential_uri: "https://accounts.google.com/o/oauth2/token",
+          audience: "https://accounts.google.com/o/oauth2/token",
+          scope: @scope,
+          issuer: email,
+          signing_key: key
+        )
+      end
+
+      def get_auth_from_compute_engine
+        Google::Auth::GCECredentials.new
+      end
+
+      def get_auth_from_json_key
+        json_key = @auth_options[:json_key]
+
+        begin
+          File.open(json_key) do |f|
+            Google::Auth::ServiceAccountCredentials.make_creds(json_key_io: f, scope: @scope)
+          end
+        rescue Errno::ENOENT
+          key = StringIO.new(json_key)
+          Google::Auth::ServiceAccountCredentials.make_creds(json_key_io: key, scope: @scope)
+        end
+      end
+
+      def get_auth_from_application_default
+        Google::Auth.get_application_default([@scope])
+      end
+    end
+  end
+end