fluent-plugin-bigquery 2.0.0.beta → 2.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: '07998acf05ddb3e647da13a4b5c734dc16f8cc77'
-  data.tar.gz: 1fce9fc906cbf72083a4f8132c0ac1d985a95d6d
+SHA256:
+  metadata.gz: 4209a2b6eaaf0b6f8ba315b6f5de6690e28fb47890aeea777bdb31889e4785ab
+  data.tar.gz: b0983fb4fa16d72059b0e679ea4ee627d19e805779fa010888fa1723354896a5
 SHA512:
-  metadata.gz: 04cfd6d3080d9424e25bd75ae1a9600259fe94ed933adceab66c02eb11afdb49eeddc393c305f0927dd64f967d1e72835fde9566cd54b2e53805e85ffe7a1516
-  data.tar.gz: 8de74527cf12be2c6553e4a582cc25c47a1773cdc165800f212aae563f7ffa048679260515a51f55e244b641b968badeaa4349cf4369ad2363d22aff1c1cbe7d
+  metadata.gz: a6fc6891eda12bbc1272af7af9c4e8d48e588bc7ef65153b3a7524e39468baebb8fdb925856d1850bbda12fed5d33865faa56542503f76fdf724a18937c7d56e
+  data.tar.gz: fff0599b6a838cb4ff233ba9585b558ff733eed8063c1cf36ee08aaacb9b3c2ca1bce4d13db2a51ecc72c398ba751a18b2856a6348f43738ee8ca366becdea61

data/.github/workflows/linux.yml

@@ -0,0 +1,31 @@
+name: Testing on Ubuntu
+on:
+  - push
+  - pull_request
+jobs:
+  build:
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        ruby:
+          - 2.6
+          - 2.7
+          - 3.0
+          - 3.1
+        os:
+          - ubuntu-latest
+    name: Ruby ${{ matrix.ruby }} unit testing on ${{ matrix.os }}
+    steps:
+    - uses: actions/checkout@v2
+    - uses: ruby/setup-ruby@v1
+      with:
+        ruby-version: ${{ matrix.ruby }}
+    - name: unit testing
+      env:
+        CI: true
+      run: |
+        ruby -v
+        gem install bundler rake
+        bundle install --jobs 4 --retry 3
+        bundle exec rake test

data/.github/workflows/windows.yml

@@ -0,0 +1,27 @@
+name: Testing on Windows
+on:
+  - push
+  - pull_request
+jobs:
+  build:
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        ruby: [ '2.6', '2.7', '3.0', '3.1' ]
+        os:
+          - windows-latest
+    name: Ruby ${{ matrix.ruby }} unit testing on ${{ matrix.os }}
+    steps:
+    - uses: actions/checkout@v2
+    - uses: ruby/setup-ruby@v1
+      with:
+        ruby-version: ${{ matrix.ruby }}
+    - name: unit testing
+      env:
+        CI: true
+      run: |
+        ruby -v
+        gem install bundler rake
+        bundle install --jobs 4 --retry 3
+        bundle exec rake test

data/README.md

@@ -1,6 +1,12 @@
 # fluent-plugin-bigquery
 
-**This README is for v2.0.0.beta. but it is not released yet. sorry.**
+## Notice
+
+We will transfer fluent-plugin-bigquery repository to [fluent-plugins-nursery](https://github.com/fluent-plugins-nursery) organization.
+It does not change maintenance plan.
+The main purpose is that it solves mismatch between maintainers and current organization. 
+
+---
 
 [Fluentd](http://fluentd.org) output plugin to load/insert data into Google BigQuery.
 
@@ -18,11 +24,13 @@
 Current version of this plugin supports Google API with Service Account Authentication, but does not support
 OAuth flow for installed applications.
 
-## Version Information
-v1.0.0 or later supports fluentd-0.14.0 or later.
-If you use fluentd-0.12.x, please use v0.4.x.
+## Support Version
 
-I recommend to update fluentd version to v0.14.x or later.
+| plugin version | fluentd version | ruby version |
+| :-----------   | :-----------    | :----------- |
+| v0.4.x         | 0.12.x          | 2.0 or later |
+| v1.x.x         | 0.14.x or later | 2.2 or later |
+| v2.x.x         | 0.14.x or later | 2.3 or later |
 
 ## With docker image
 If you use official alpine based fluentd docker image (https://github.com/fluent/fluentd-docker-image),
@@ -37,28 +45,31 @@ Because embbeded gem dependency sometimes restricts ruby environment.
 
 #### common
 
-| name                                   | type          | required?                                    | placeholder? | default                    | description                                                                                            |
-| :------------------------------------- | :------------ | :-----------                                 | :----------  | :------------------------- | :-----------------------                                                                               |
-| auth_method                            | enum          | yes                                          | no           | private_key                | `private_key` or `json_key` or `compute_engine` or `application_default`                               |
-| email                                  | string        | yes (private_key)                            | no           | nil                        | GCP Service Account Email                                                                              |
-| private_key_path                       | string        | yes (private_key)                            | no           | nil                        | GCP Private Key file path                                                                              |
-| private_key_passphrase                 | string        | yes (private_key)                            | no           | nil                        | GCP Private Key Passphrase                                                                             |
-| json_key                               | string        | yes (json_key)                               | no           | nil                        | GCP JSON Key file path or JSON Key string                                                              |
-| project                                | string        | yes                                          | yes          | nil                        |                                                                                                        |
-| dataset                                | string        | yes                                          | yes          | nil                        |                                                                                                        |
-| table                                  | string        | yes (either `tables`)                        | yes          | nil                        |                                                                                                        |
-| tables                                 | array(string) | yes (either `table`)                         | yes          | nil                        | can set multi table names splitted by `,`                                                              |
-| auto_create_table                      | bool          | no                                           | no           | false                      | If true, creates table automatically                                                                   |
-| ignore_unknown_values                  | bool          | no                                           | no           | false                      | Accept rows that contain values that do not match the schema. The unknown values are ignored.          |
-| schema                                 | array         | yes (either `fetch_schema` or `schema_path`) | no           | nil                        | Schema Definition. It is formatted by JSON.                                                            |
-| schema_path                            | string        | yes (either `fetch_schema`)                  | no           | nil                        | Schema Definition file path. It is formatted by JSON.                                                  |
-| fetch_schema                           | bool          | yes (either `schema_path`)                   | no           | false                      | If true, fetch table schema definition from Bigquery table automatically.                              |
-| fetch_schema_table                     | string        | no                                           | yes          | nil                        | If set, fetch table schema definition from this table, If fetch_schema is false, this param is ignored |
-| schema_cache_expire                    | integer       | no                                           | no           | 600                        | Value is second. If current time is after expiration interval, re-fetch table schema definition.       |
-| request_timeout_sec                    | integer       | no                                           | no           | nil                        | Bigquery API response timeout                                                                          |
-| request_open_timeout_sec               | integer       | no                                           | no           | 60                         | Bigquery API connection, and request timeout. If you send big data to Bigquery, set large value.       |
-| time_partitioning_type                 | enum          | no (either day)                              | no           | nil                        | Type of bigquery time partitioning feature(experimental feature on BigQuery).                          |
-| time_partitioning_expiration           | time          | no                                           | no           | nil                        | Expiration milliseconds for bigquery time partitioning. (experimental feature on BigQuery)             |
+| name                                          | type          | required?                                    | placeholder? | default                    | description                                                                                            |
+| :-------------------------------------------- | :------------ | :-----------                                 | :----------  | :------------------------- | :-----------------------                                                                               |
+| auth_method                                   | enum          | yes                                          | no           | private_key                | `private_key` or `json_key` or `compute_engine` or `application_default`                               |
+| email                                         | string        | yes (private_key)                            | no           | nil                        | GCP Service Account Email                                                                              |
+| private_key_path                              | string        | yes (private_key)                            | no           | nil                        | GCP Private Key file path                                                                              |
+| private_key_passphrase                        | string        | yes (private_key)                            | no           | nil                        | GCP Private Key Passphrase                                                                             |
+| json_key                                      | string        | yes (json_key)                               | no           | nil                        | GCP JSON Key file path or JSON Key string                                                              |
+| location                                      | string        | no                                           | no           | nil                        | BigQuery Data Location. The geographic location of the job. Required except for US and EU.             |
+| project                                       | string        | yes                                          | yes          | nil                        |                                                                                                        |
+| dataset                                       | string        | yes                                          | yes          | nil                        |                                                                                                        |
+| table                                         | string        | yes (either `tables`)                        | yes          | nil                        |                                                                                                        |
+| tables                                        | array(string) | yes (either `table`)                         | yes          | nil                        | can set multi table names splitted by `,`                                                              |
+| auto_create_table                             | bool          | no                                           | no           | false                      | If true, creates table automatically                                                                   |
+| ignore_unknown_values                         | bool          | no                                           | no           | false                      | Accept rows that contain values that do not match the schema. The unknown values are ignored.          |
+| schema                                        | array         | yes (either `fetch_schema` or `schema_path`) | no           | nil                        | Schema Definition. It is formatted by JSON.                                                            |
+| schema_path                                   | string        | yes (either `fetch_schema`)                  | yes          | nil                        | Schema Definition file path. It is formatted by JSON.                                                  |
+| fetch_schema                                  | bool          | yes (either `schema_path`)                   | no           | false                      | If true, fetch table schema definition from Bigquery table automatically.                              |
+| fetch_schema_table                            | string        | no                                           | yes          | nil                        | If set, fetch table schema definition from this table, If fetch_schema is false, this param is ignored |
+| schema_cache_expire                           | integer       | no                                           | no           | 600                        | Value is second. If current time is after expiration interval, re-fetch table schema definition.       |
+| request_timeout_sec                           | integer       | no                                           | no           | nil                        | Bigquery API response timeout                                                                          |
+| request_open_timeout_sec                      | integer       | no                                           | no           | 60                         | Bigquery API connection, and request timeout. If you send big data to Bigquery, set large value.       |
+| time_partitioning_type                        | enum          | no (either day)                              | no           | nil                        | Type of bigquery time partitioning feature.                                                            |
+| time_partitioning_field                       | string        | no                                           | no           | nil                        | Field used to determine how to create a time-based partition.                                          |
+| time_partitioning_expiration                  | time          | no                                           | no           | nil                        | Expiration milliseconds for bigquery time partitioning.                                                |
+| clustering_fields                             | array(string) | no                                           | no           | nil                        | One or more fields on which data should be clustered. The order of the specified columns determines the sort order of the data. |
 
 #### bigquery_insert
 
@@ -69,6 +80,7 @@ Because embbeded gem dependency sometimes restricts ruby environment.
 | insert_id_field                        | string        | no           | no           | nil                        | Use key as `insert_id` of Streaming Insert API parameter. see. https://docs.fluentd.org/v1.0/articles/api-plugin-helper-record_accessor                                                    |
 | add_insert_timestamp                   | string        | no           | no           | nil                        | Adds a timestamp column just before sending the rows to BigQuery, so that buffering time is not taken into account. Gives a field in BigQuery which represents the insert time of the row. |
 | allow_retry_insert_errors              | bool          | no           | no           | false                      | Retry to insert rows when an insertErrors occurs. There is a possibility that rows are inserted in duplicate.                                                                              |
+| require_partition_filter    | bool          | no                                           | no           | false                      | If true, queries over this table require a partition filter that can be used for partition elimination to be specified. |
 
 #### bigquery_load
 
@@ -431,7 +443,7 @@ Use placeholder.
 
 ```apache
 <match dummy>
-  @type bigquery_insert
+  @type bigquery_load
 
   ...
   table   accesslog$%Y%m%d
@@ -444,6 +456,8 @@ Use placeholder.
 ```
 
 But, Dynamic table creating doesn't support date partitioned table yet.
+And streaming insert is not allowed to insert with `$%Y%m%d` suffix.
+If you use date partitioned table with streaming insert, Please omit `$%Y%m%d` suffix from `table`.
 
 ### Dynamic table creating
 
@@ -465,6 +479,8 @@ NOTE: `auto_create_table` option cannot be used with `fetch_schema`. You should
 </match>
 ```
 
+Also, you can create clustered table by using `clustering_fields`.
+
 ### Table schema
 
 There are three methods to describe the schema of the target table.

data/lib/fluent/plugin/bigquery/errors.rb

@@ -7,10 +7,9 @@ module Fluent
       RETRYABLE_STATUS_CODE = [500, 502, 503, 504]
 
       class << self
-        def wrap(google_api_error, message = nil, force_unretryable: false)
-          e = google_api_error
-          return UnRetryableError.new(message, e) if force_unretryable
-
+        # @param e [Google::Apis::Error]
+        # @param message [String]
+        def wrap(e, message = nil)
           if retryable_error?(e)
             RetryableError.new(message, e)
           else
@@ -18,12 +17,9 @@ module Fluent
           end
         end
 
-        def retryable_error?(google_api_error)
-          e = google_api_error
-          reason = e.respond_to?(:reason) ? e.reason : nil
-
-          retryable_error_reason?(reason) ||
-            (e.is_a?(Google::Apis::ServerError) && RETRYABLE_STATUS_CODE.include?(e.status_code))
+        # @param e [Google::Apis::Error]
+        def retryable_error?(e)
+          e.is_a?(Google::Apis::ServerError) && RETRYABLE_STATUS_CODE.include?(e.status_code)
         end
 
         def retryable_error_reason?(reason)

data/lib/fluent/plugin/bigquery/schema.rb

@@ -86,6 +86,16 @@ module Fluent
       end
     end
 
+    class NumericFieldSchema < FieldSchema
+      def type
+        :numeric
+      end
+
+      def format_one(value)
+        value.to_s
+      end
+    end
+
     class BooleanFieldSchema < FieldSchema
       def type
         :boolean
@@ -169,6 +179,7 @@ module Fluent
         string: StringFieldSchema,
         integer: IntegerFieldSchema,
         float: FloatFieldSchema,
+        numeric: NumericFieldSchema,
         boolean: BooleanFieldSchema,
         timestamp: TimestampFieldSchema,
         date: DateFieldSchema,

data/lib/fluent/plugin/bigquery/version.rb

@@ -1,5 +1,5 @@
 module Fluent
   module BigQueryPlugin
-    VERSION = "2.0.0.beta".freeze
+    VERSION = "2.3.0".freeze
   end
 end

data/lib/fluent/plugin/bigquery/writer.rb

@@ -34,12 +34,9 @@ module Fluent
             }
           }
 
-          if @options[:time_partitioning_type]
-            definition[:time_partitioning] = {
-              type: @options[:time_partitioning_type].to_s.upcase,
-              expiration_ms: @options[:time_partitioning_expiration] ? @options[:time_partitioning_expiration] * 1000 : nil
-            }.select { |_, value| !value.nil? }
-          end
+          definition.merge!(time_partitioning: time_partitioning) if time_partitioning
+          definition.merge!(require_partition_filter: require_partition_filter) if require_partition_filter
+          definition.merge!(clustering: clustering) if clustering
           client.insert_table(project, dataset, definition, {})
           log.debug "create table", project_id: project, dataset: dataset, table: table_id
         rescue Google::Apis::ServerError, Google::Apis::ClientError, Google::Apis::AuthorizationError => e
@@ -50,10 +47,9 @@ module Fluent
             return
           end
 
-          reason = e.respond_to?(:reason) ? e.reason : nil
-          log.error "tables.insert API", project_id: project, dataset: dataset, table: table_id, code: e.status_code, message: message, reason: reason
+          log.error "tables.insert API", project_id: project, dataset: dataset, table: table_id, code: e.status_code, message: message
 
-          if Fluent::BigQuery::Error.retryable_error_reason?(reason) && create_table_retry_count < create_table_retry_limit
+          if create_table_retry_count < create_table_retry_limit
             sleep create_table_retry_wait
             create_table_retry_wait *= 2
             create_table_retry_count += 1
@@ -76,14 +72,19 @@ module Fluent
         nil
       end
 
-      def insert_rows(project, dataset, table_id, rows, template_suffix: nil)
+      def insert_rows(project, dataset, table_id, rows, schema, template_suffix: nil)
         body = {
           rows: rows,
           skip_invalid_rows: @options[:skip_invalid_rows],
           ignore_unknown_values: @options[:ignore_unknown_values],
         }
         body.merge!(template_suffix: template_suffix) if template_suffix
-        res = client.insert_all_table_data(project, dataset, table_id, body, {})
+
+        if @options[:auto_create_table]
+          res = insert_all_table_data_with_create_table(project, dataset, table_id, body, schema)
+        else
+          res = client.insert_all_table_data(project, dataset, table_id, body, {})
+        end
         log.debug "insert rows", project_id: project, dataset: dataset, table: table_id, count: rows.size
 
         if res.insert_errors && !res.insert_errors.empty?
@@ -100,8 +101,7 @@ module Fluent
           end
         end
       rescue Google::Apis::ServerError, Google::Apis::ClientError, Google::Apis::AuthorizationError => e
-        reason = e.respond_to?(:reason) ? e.reason : nil
-        error_data = { project_id: project, dataset: dataset, table: table_id, code: e.status_code, message: e.message, reason: reason }
+        error_data = { project_id: project, dataset: dataset, table: table_id, code: e.status_code, message: e.message }
         wrapped = Fluent::BigQuery::Error.wrap(e)
         if wrapped.retryable?
           log.warn "tabledata.insertAll API", error_data
@@ -131,9 +131,6 @@ module Fluent
                 dataset_id: dataset,
                 table_id: table_id,
               },
-              schema: {
-                fields: fields.to_a,
-              },
               write_disposition: "WRITE_APPEND",
               source_format: source_format,
               ignore_unknown_values: @options[:ignore_unknown_values],
@@ -143,17 +140,19 @@ module Fluent
         }
 
         job_id = create_job_id(chunk_id_hex, dataset, table_id, fields.to_a) if @options[:prevent_duplicate_load]
-        configuration[:configuration][:load].merge!(create_disposition: "CREATE_NEVER") if @options[:time_partitioning_type]
         configuration.merge!({job_reference: {project_id: project, job_id: job_id}}) if job_id
 
-        # If target table is already exist, omit schema configuration.
-        # Because schema changing is easier.
         begin
-          if client.get_table(project, dataset, table_id)
-            configuration[:configuration][:load].delete(:schema)
+          # Check table existance
+          client.get_table(project, dataset, table_id)
+        rescue Google::Apis::ServerError, Google::Apis::ClientError, Google::Apis::AuthorizationError => e
+          if e.status_code == 404 && /Not Found: Table/i =~ e.message
+            raise Fluent::BigQuery::UnRetryableError.new("Table is not found") unless @options[:auto_create_table]
+            raise Fluent::BigQuery::UnRetryableError.new("Schema is empty") if fields.empty?
+            configuration[:configuration][:load].merge!(schema: {fields: fields.to_a})
+            configuration[:configuration][:load].merge!(time_partitioning: time_partitioning) if time_partitioning
+            configuration[:configuration][:load].merge!(clustering: clustering) if clustering
           end
-        rescue Google::Apis::ServerError, Google::Apis::ClientError, Google::Apis::AuthorizationError
-          raise Fluent::BigQuery::UnRetryableError.new("Schema is empty") if fields.empty?
         end
 
         res = client.insert_job(
@@ -166,19 +165,7 @@ module Fluent
         )
         JobReference.new(chunk_id, chunk_id_hex, project, dataset, table_id, res.job_reference.job_id)
       rescue Google::Apis::ServerError, Google::Apis::ClientError, Google::Apis::AuthorizationError => e
-        reason = e.respond_to?(:reason) ? e.reason : nil
-        log.error "job.load API", project_id: project, dataset: dataset, table: table_id, code: e.status_code, message: e.message, reason: reason
-
-        if @options[:auto_create_table] && e.status_code == 404 && /Not Found: Table/i =~ e.message
-          # Table Not Found: Auto Create Table
-          create_table(
-            project,
-            dataset,
-            table_id,
-            fields,
-          )
-          raise "table created. send rows next time."
-        end
+        log.error "job.load API", project_id: project, dataset: dataset, table: table_id, code: e.status_code, message: e.message
 
         if job_id && e.status_code == 409 && e.message =~ /Job/ # duplicate load job
           return JobReference.new(chunk_id, chunk_id_hex, project, dataset, table_id, job_id)
@@ -190,8 +177,9 @@ module Fluent
       def fetch_load_job(job_reference)
         project = job_reference.project_id
         job_id = job_reference.job_id
+        location = @options[:location]
 
-        res = client.get_job(project, job_id)
+        res = client.get_job(project, job_id, location: location)
         log.debug "load job fetched", id: job_id, state: res.status.state, **job_reference.as_hash(:project_id, :dataset_id, :table_id)
 
         if res.status.state == "DONE"
@@ -227,9 +215,10 @@ module Fluent
           end
         end
 
+        # `stats` can be nil if we receive a warning like "Warning: Load job succeeded with data imported, however statistics may be lost due to internal error."
         stats = response.statistics.load
         duration = (response.statistics.end_time - response.statistics.creation_time) / 1000.0
-        log.debug "load job finished", id: job_id, state: response.status.state, input_file_bytes: stats.input_file_bytes, input_files: stats.input_files, output_bytes: stats.output_bytes, output_rows: stats.output_rows, bad_records: stats.bad_records, duration: duration.round(2), project_id: project, dataset: dataset, table: table_id
+        log.debug "load job finished", id: job_id, state: response.status.state, input_file_bytes: stats&.input_file_bytes, input_files: stats&.input_files, output_bytes: stats&.output_bytes, output_rows: stats&.output_rows, bad_records: stats&.bad_records, duration: duration.round(2), project_id: project, dataset: dataset, table: table_id
         @num_errors_per_chunk.delete(chunk_id_hex)
       end
 
@@ -315,6 +304,64 @@ module Fluent
           "NEWLINE_DELIMITED_JSON"
         end
       end
+
+      def time_partitioning
+        return @time_partitioning if instance_variable_defined?(:@time_partitioning)
+
+        if @options[:time_partitioning_type]
+          @time_partitioning = {
+            type: @options[:time_partitioning_type].to_s.upcase,
+            field: @options[:time_partitioning_field] ? @options[:time_partitioning_field].to_s : nil,
+            expiration_ms: @options[:time_partitioning_expiration] ? @options[:time_partitioning_expiration] * 1000 : nil,
+          }.reject { |_, v| v.nil? }
+        else
+          @time_partitioning
+        end
+      end
+
+      def require_partition_filter
+        return @require_partition_filter if instance_variable_defined?(:@require_partition_filter)
+
+        if @options[:require_partition_filter]
+          @require_partition_filter = @options[:require_partition_filter]
+        else
+          @require_partition_filter
+        end
+      end
+
+      def clustering
+        return @clustering if instance_variable_defined?(:@clustering)
+
+        if @options[:clustering_fields]
+          @clustering = {
+            fields: @options[:clustering_fields]
+          }
+        else
+          @clustering
+        end
+      end
+
+      def insert_all_table_data_with_create_table(project, dataset, table_id, body, schema)
+        try_count ||= 1
+        res = client.insert_all_table_data(project, dataset, table_id, body, {})
+      rescue Google::Apis::ClientError => e
+        if e.status_code == 404 && /Not Found: Table/i =~ e.message
+          if try_count == 1
+            # Table Not Found: Auto Create Table
+            create_table(project, dataset, table_id, schema)
+          elsif try_count > 10
+            raise "A new table was created but it is not found."
+          end
+
+          # Retry to insert several times because the created table is not visible from Streaming insert for a little while
+          # cf. https://cloud.google.com/bigquery/troubleshooting-errors#metadata-errors-for-streaming-inserts
+          try_count += 1
+          sleep 5
+          log.debug "Retry to insert rows", project_id: project, dataset: dataset, table: table_id
+          retry
+        end
+        raise
+      end
     end
   end
 end

data/lib/fluent/plugin/out_bigquery_base.rb

@@ -29,6 +29,9 @@ module Fluent
       config_param :private_key_path, :string, default: nil
       config_param :private_key_passphrase, :string, default: 'notasecret', secret: true
       config_param :json_key, default: nil, secret: true
+      # The geographic location of the job. Required except for US and EU.
+      # https://github.com/googleapis/google-api-ruby-client/blob/master/generated/google/apis/bigquery_v2/service.rb#L350
+      config_param :location, :string, default: nil
 
       # see as simple reference
       #   https://github.com/abronte/BigQuery/blob/master/lib/bigquery.rb
@@ -67,8 +70,12 @@ module Fluent
 
       ## Partitioning
       config_param :time_partitioning_type, :enum, list: [:day], default: nil
+      config_param :time_partitioning_field, :string, default: nil
       config_param :time_partitioning_expiration, :time, default: nil
 
+      ## Clustering
+      config_param :clustering_fields, :array, default: nil
+
       ## Formatter
       config_section :format do
         config_set_default :@type, 'json'
@@ -104,9 +111,6 @@ module Fluent
         if @schema
           @table_schema.load_schema(@schema)
         end
-        if @schema_path
-          @table_schema.load_schema(MultiJson.load(File.read(@schema_path)))
-        end
 
         formatter_config = conf.elements("format")[0]
         @formatter = formatter_create(usage: 'out_bigquery_for_insert', default_type: 'json', conf: formatter_config)
@@ -119,6 +123,7 @@ module Fluent
         @tables_mutex = Mutex.new
         @fetched_schemas = {}
         @last_fetch_schema_time = Hash.new(0)
+        @read_schemas = {}
       end
 
       def multi_workers_ready?
@@ -130,6 +135,7 @@ module Fluent
           private_key_path: @private_key_path, private_key_passphrase: @private_key_passphrase,
           email: @email,
           json_key: @json_key,
+          location: @location,
           source_format: @source_format,
           skip_invalid_rows: @skip_invalid_rows,
           ignore_unknown_values: @ignore_unknown_values,
@@ -138,7 +144,10 @@ module Fluent
           prevent_duplicate_load: @prevent_duplicate_load,
           auto_create_table: @auto_create_table,
           time_partitioning_type: @time_partitioning_type,
+          time_partitioning_field: @time_partitioning_field,
           time_partitioning_expiration: @time_partitioning_expiration,
+          require_partition_filter: @require_partition_filter,
+          clustering_fields: @clustering_fields,
           timeout_sec: @request_timeout_sec,
           open_timeout_sec: @request_open_timeout_sec,
         })
@@ -151,6 +160,8 @@ module Fluent
         schema =
           if @fetch_schema
             fetch_schema(meta)
+          elsif @schema_path
+            read_schema(meta)
           else
             @table_schema
           end
@@ -182,7 +193,7 @@ module Fluent
             table_schema.load_schema(schema)
             @fetched_schemas["#{project}.#{dataset}.#{table_id}"] = table_schema
           else
-            if @fetched_schemas["#{project}.#{dataset}.#{table_id}"].empty?
+            if @fetched_schemas["#{project}.#{dataset}.#{table_id}"].nil?
               raise "failed to fetch schema from bigquery"
             else
               log.warn "#{table_id} uses previous schema"
@@ -199,9 +210,26 @@ module Fluent
         extract_placeholders(@fetch_schema_table || @tablelist[0], metadata)
       end
 
+      def read_schema(metadata)
+        schema_path = read_schema_target_path(metadata)
+
+        unless @read_schemas[schema_path]
+          table_schema = Fluent::BigQuery::RecordSchema.new("record")
+          table_schema.load_schema(MultiJson.load(File.read(schema_path)))
+          @read_schemas[schema_path] = table_schema
+        end
+        @read_schemas[schema_path]
+      end
+
+      def read_schema_target_path(metadata)
+        extract_placeholders(@schema_path, metadata)
+      end
+
       def get_schema(project, dataset, metadata)
         if @fetch_schema
           @fetched_schemas["#{project}.#{dataset}.#{fetch_schema_target_table(metadata)}"] || fetch_schema(metadata)
+        elsif @schema_path
+          @read_schemas[read_schema_target_path(metadata)] || read_schema(metadata)
         else
           @table_schema
         end

data/lib/fluent/plugin/out_bigquery_insert.rb

@@ -29,6 +29,9 @@ module Fluent
       # If insert_id_field is not specified, true means to allow duplicate rows
       config_param :allow_retry_insert_errors, :bool, default: false
 
+      ## RequirePartitionFilter
+      config_param :require_partition_filter, :bool, default: false
+
       ## Buffer
       config_section :buffer do
         config_set_default :@type, "memory"
@@ -96,14 +99,8 @@ module Fluent
       end
 
       def insert(project, dataset, table_id, rows, schema, template_suffix)
-        writer.insert_rows(project, dataset, table_id, rows, template_suffix: template_suffix)
+        writer.insert_rows(project, dataset, table_id, rows, schema, template_suffix: template_suffix)
       rescue Fluent::BigQuery::Error => e
-        if @auto_create_table && e.status_code == 404 && /Not Found: Table/i =~ e.message
-          # Table Not Found: Auto Create Table
-          writer.create_table(project, dataset, table_id, schema)
-          raise "table created. send rows next time."
-        end
-
         raise if e.retryable?
 
         if @secondary

data/lib/fluent/plugin/out_bigquery_load.rb

@@ -196,6 +196,7 @@ module Fluent
         rescue => e
           log.error("unexpected error while polling", error: e)
           log.error_backtrace
+          rollback_write(job_reference.chunk_id)
         end
       end
 

data/test/plugin/test_out_bigquery_base.rb

@@ -147,33 +147,6 @@ class BigQueryBaseOutputTest < Test::Unit::TestCase
     assert driver.instance.writer.client.is_a?(Google::Apis::BigqueryV2::BigqueryService)
   end
 
-  def test_configure_auth_json_key_as_file_raise_permission_error
-    json_key_path = 'test/plugin/testdata/json_key.json'
-    json_key_path_dir = File.dirname(json_key_path)
-
-    begin
-      File.chmod(0000, json_key_path_dir)
-
-      driver = create_driver(%[
-        table foo
-        auth_method json_key
-        json_key #{json_key_path}
-        project yourproject_id
-        dataset yourdataset_id
-        schema [
-          {"name": "time", "type": "INTEGER"},
-          {"name": "status", "type": "INTEGER"},
-          {"name": "bytes", "type": "INTEGER"}
-        ]
-      ])
-      assert_raises(Errno::EACCES) do
-        driver.instance.writer.client
-      end
-    ensure
-      File.chmod(0755, json_key_path_dir)
-    end
-  end
-
   def test_configure_auth_json_key_as_string
     json_key = '{"private_key": "X", "client_email": "' + 'x' * 255 + '@developer.gserviceaccount.com"}'
     json_key_io = StringIO.new(json_key)
@@ -199,6 +172,8 @@ class BigQueryBaseOutputTest < Test::Unit::TestCase
   end
 
   def test_configure_auth_application_default
+    omit "This testcase depends on some environment variables." if ENV["CI"] == "true"
+
     driver = create_driver(%[
       table foo
       auth_method application_default
@@ -576,4 +551,24 @@ class BigQueryBaseOutputTest < Test::Unit::TestCase
     assert_equal :string, table_schema["argv"].type
     assert_equal :repeated, table_schema["argv"].mode
   end
+
+  def test_resolve_schema_path_with_placeholder
+    now = Time.now.to_i
+    driver = create_driver(<<-CONFIG)
+      table ${tag}_%Y%m%d
+      auth_method json_key
+      json_key jsonkey.josn
+      project yourproject_id
+      dataset yourdataset_id
+      schema_path ${tag}.schema
+
+      <buffer tag, time>
+        timekey 1d
+      </buffer>
+    CONFIG
+
+    metadata = Fluent::Plugin::Buffer::Metadata.new(now, "foo", {})
+
+    assert_equal "foo.schema", driver.instance.read_schema_target_path(metadata)
+  end
 end

data/test/plugin/test_out_bigquery_insert.rb

@@ -5,6 +5,8 @@ class BigQueryInsertOutputTest < Test::Unit::TestCase
     Fluent::Test.setup
   end
 
+  SCHEMA_PATH = File.join(File.dirname(__FILE__), "testdata", "apache.schema")
+
   CONFIG = %[
     table foo
     email foo@bar.example
@@ -121,7 +123,6 @@ class BigQueryInsertOutputTest < Test::Unit::TestCase
     driver = create_driver
 
     stub_writer do |writer|
-      mock.proxy(writer).insert_rows('yourproject_id', 'yourdataset_id', 'foo', [{json: hash_including(entry)}], template_suffix: nil)
       mock(writer.client).insert_all_table_data('yourproject_id', 'yourdataset_id', 'foo', {
         rows: [{json: hash_including(entry)}],
         skip_invalid_rows: false,
@@ -261,7 +262,7 @@ class BigQueryInsertOutputTest < Test::Unit::TestCase
 
     driver.instance_start
     tag, time, record = "tag", Time.now.to_i, {"a" => "b"}
-    metadata = driver.instance.metadata_for_test(tag, time, record)
+    metadata = Fluent::Plugin::Buffer::Metadata.new(tag, time, record)
     chunk = driver.instance.buffer.generate_chunk(metadata).tap do |c|
       c.append([driver.instance.format(tag, time, record)])
     end
@@ -345,11 +346,27 @@ class BigQueryInsertOutputTest < Test::Unit::TestCase
       schema_path #{File.join(File.dirname(__FILE__), "testdata", "apache.schema")}
     CONFIG
 
+    schema_fields = Fluent::BigQuery::Helper.deep_symbolize_keys(MultiJson.load(File.read(SCHEMA_PATH)))
+
     stub_writer do |writer|
-      mock(writer).insert_rows('yourproject_id', 'yourdataset_id', 'foo', [{json: Fluent::BigQuery::Helper.deep_symbolize_keys(message)}], template_suffix: nil) do
-        raise Fluent::BigQuery::RetryableError.new(nil, Google::Apis::ServerError.new("Not found: Table yourproject_id:yourdataset_id.foo", status_code: 404, body: "Not found: Table yourproject_id:yourdataset_id.foo"))
-      end
-      mock(writer).create_table('yourproject_id', 'yourdataset_id', 'foo', driver.instance.instance_variable_get(:@table_schema))
+      body = {
+        rows: [{json: Fluent::BigQuery::Helper.deep_symbolize_keys(message)}],
+        skip_invalid_rows: false,
+        ignore_unknown_values: false,
+      }
+      mock(writer.client).insert_all_table_data('yourproject_id', 'yourdataset_id', 'foo', body, {}) do
+        raise Google::Apis::ClientError.new("notFound: Not found: Table yourproject_id:yourdataset_id.foo", status_code: 404)
+      end.at_least(1)
+      mock(writer).sleep(instance_of(Numeric)) { nil }.at_least(1)
+
+      mock(writer.client).insert_table('yourproject_id', 'yourdataset_id', {
+        table_reference: {
+          table_id: 'foo',
+        },
+        schema: {
+          fields: schema_fields,
+        },
+      }, {})
     end
 
     assert_raise(RuntimeError) do
@@ -401,14 +418,131 @@ class BigQueryInsertOutputTest < Test::Unit::TestCase
       schema_path #{File.join(File.dirname(__FILE__), "testdata", "apache.schema")}
 
       time_partitioning_type day
+      time_partitioning_field time
       time_partitioning_expiration 1h
+
+      require_partition_filter true
     CONFIG
 
+    schema_fields = Fluent::BigQuery::Helper.deep_symbolize_keys(MultiJson.load(File.read(SCHEMA_PATH)))
+
     stub_writer do |writer|
-      mock(writer).insert_rows('yourproject_id', 'yourdataset_id', 'foo', [message], template_suffix: nil) do
-        raise Fluent::BigQuery::RetryableError.new(nil, Google::Apis::ServerError.new("Not found: Table yourproject_id:yourdataset_id.foo", status_code: 404, body: "Not found: Table yourproject_id:yourdataset_id.foo"))
+      body = {
+        rows: [message],
+        skip_invalid_rows: false,
+        ignore_unknown_values: false,
+      }
+      mock(writer.client).insert_all_table_data('yourproject_id', 'yourdataset_id', 'foo', body, {}) do
+        raise Google::Apis::ClientError.new("notFound: Not found: Table yourproject_id:yourdataset_id.foo", status_code: 404)
+      end.at_least(1)
+      mock(writer).sleep(instance_of(Numeric)) { nil }.at_least(1)
+
+      mock(writer.client).insert_table('yourproject_id', 'yourdataset_id', {
+        table_reference: {
+          table_id: 'foo',
+        },
+        schema: {
+          fields: schema_fields,
+        },
+        time_partitioning: {
+          type: 'DAY',
+          field: 'time',
+          expiration_ms: 3600000,
+        },
+        require_partition_filter: true,
+      }, {})
+    end
+
+    assert_raise(RuntimeError) do
+      driver.run do
+        driver.feed("tag", Fluent::EventTime.now, message[:json])
       end
-      mock(writer).create_table('yourproject_id', 'yourdataset_id', 'foo', driver.instance.instance_variable_get(:@table_schema))
+    end
+  end
+
+  def test_auto_create_clustered_table_by_bigquery_api
+    now = Time.now
+    message = {
+      json: {
+        time: now.to_i,
+        request: {
+          vhost: "bar",
+          path: "/path/to/baz",
+          method: "GET",
+          protocol: "HTTP/1.0",
+          agent: "libwww",
+          referer: "http://referer.example",
+          time: (now - 1).to_f,
+          bot_access: true,
+          loginsession: false,
+        },
+        remote: {
+          host: "remote.example",
+          ip: "192.168.1.1",
+          user: "nagachika",
+        },
+        response: {
+          status: 200,
+          bytes: 72,
+        },
+      }
+    }
+
+    driver = create_driver(<<-CONFIG)
+      table foo
+      email foo@bar.example
+      private_key_path /path/to/key
+      project yourproject_id
+      dataset yourdataset_id
+
+      time_format %s
+      time_field  time
+
+      auto_create_table true
+      schema_path #{File.join(File.dirname(__FILE__), "testdata", "apache.schema")}
+
+      time_partitioning_type day
+      time_partitioning_field time
+      time_partitioning_expiration 1h
+
+      clustering_fields [
+        "time",
+        "vhost"
+      ]
+    CONFIG
+
+    schema_fields = Fluent::BigQuery::Helper.deep_symbolize_keys(MultiJson.load(File.read(SCHEMA_PATH)))
+
+    stub_writer do |writer|
+      body = {
+        rows: [message],
+        skip_invalid_rows: false,
+        ignore_unknown_values: false,
+      }
+      mock(writer.client).insert_all_table_data('yourproject_id', 'yourdataset_id', 'foo', body, {}) do
+        raise Google::Apis::ClientError.new("notFound: Not found: Table yourproject_id:yourdataset_id.foo", status_code: 404)
+      end.at_least(1)
+      mock(writer).sleep(instance_of(Numeric)) { nil }.at_least(1)
+
+      mock(writer.client).insert_table('yourproject_id', 'yourdataset_id', {
+        table_reference: {
+          table_id: 'foo',
+        },
+        schema: {
+          fields: schema_fields,
+        },
+        time_partitioning: {
+          type: 'DAY',
+          field: 'time',
+          expiration_ms: 3600000,
+        },
+        clustering: {
+          fields: [
+            'time',
+            'vhost',
+          ],
+        },
+      }, {})
     end
 
     assert_raise(RuntimeError) do

data/test/plugin/test_out_bigquery_load.rb

@@ -39,10 +39,8 @@ class BigQueryLoadOutputTest < Test::Unit::TestCase
       writer
     end
   end
-  
-  def test_write
-    schema_fields = Fluent::BigQuery::Helper.deep_symbolize_keys(MultiJson.load(File.read(SCHEMA_PATH)))
 
+  def test_write
     response_stub = stub!
 
     driver = create_driver
@@ -60,9 +58,6 @@ class BigQueryLoadOutputTest < Test::Unit::TestCase
               dataset_id: 'yourdataset_id',
               table_id: 'foo',
             },
-            schema: {
-              fields: schema_fields,
-            },
             write_disposition: "WRITE_APPEND",
             source_format: "NEWLINE_DELIMITED_JSON",
             ignore_unknown_values: false,
@@ -99,7 +94,6 @@ class BigQueryLoadOutputTest < Test::Unit::TestCase
       schema_path #{SCHEMA_PATH}
       prevent_duplicate_load true
     CONFIG
-    schema_fields = Fluent::BigQuery::Helper.deep_symbolize_keys(MultiJson.load(File.read(SCHEMA_PATH)))
 
     response_stub = stub!
     stub_writer do |writer|
@@ -116,9 +110,6 @@ class BigQueryLoadOutputTest < Test::Unit::TestCase
               dataset_id: 'yourdataset_id',
               table_id: 'foo',
             },
-            schema: {
-              fields: schema_fields,
-            },
             write_disposition: "WRITE_APPEND",
             source_format: "NEWLINE_DELIMITED_JSON",
             ignore_unknown_values: false,
@@ -138,11 +129,10 @@ class BigQueryLoadOutputTest < Test::Unit::TestCase
 
   def test_write_with_retryable_error
     driver = create_driver
-    schema_fields = Fluent::BigQuery::Helper.deep_symbolize_keys(MultiJson.load(File.read(SCHEMA_PATH)))
 
     driver.instance_start
     tag, time, record = "tag", Time.now.to_i, {"a" => "b"}
-    metadata = driver.instance.metadata_for_test(tag, time, record)
+    metadata = Fluent::Plugin::Buffer::Metadata.new(tag, time, record)
     chunk = driver.instance.buffer.generate_chunk(metadata).tap do |c|
       c.append([driver.instance.format(tag, time, record)])
     end
@@ -158,9 +148,6 @@ class BigQueryLoadOutputTest < Test::Unit::TestCase
               dataset_id: 'yourdataset_id',
               table_id: 'foo',
             },
-            schema: {
-              fields: schema_fields,
-            },
             write_disposition: "WRITE_APPEND",
             source_format: "NEWLINE_DELIMITED_JSON",
             ignore_unknown_values: false,
@@ -171,7 +158,7 @@ class BigQueryLoadOutputTest < Test::Unit::TestCase
         stub!.job_reference.stub!.job_id { "dummy_job_id" }
       end
 
-      mock(writer.client).get_job('yourproject_id', 'dummy_job_id') do
+      mock(writer.client).get_job('yourproject_id', 'dummy_job_id', :location=>nil) do
         stub! do |s|
           s.id { 'dummy_job_id' }
           s.configuration.stub! do |_s|
@@ -225,11 +212,10 @@ class BigQueryLoadOutputTest < Test::Unit::TestCase
         utc
       </secondary>
     CONFIG
-    schema_fields = Fluent::BigQuery::Helper.deep_symbolize_keys(MultiJson.load(File.read(SCHEMA_PATH)))
 
     driver.instance_start
     tag, time, record = "tag", Time.now.to_i, {"a" => "b"}
-    metadata = driver.instance.metadata_for_test(tag, time, record)
+    metadata = Fluent::Plugin::Buffer::Metadata.new(tag, time, record)
     chunk = driver.instance.buffer.generate_chunk(metadata).tap do |c|
       c.append([driver.instance.format(tag, time, record)])
     end
@@ -245,9 +231,6 @@ class BigQueryLoadOutputTest < Test::Unit::TestCase
               dataset_id: 'yourdataset_id',
               table_id: 'foo',
             },
-            schema: {
-              fields: schema_fields,
-            },
             write_disposition: "WRITE_APPEND",
             source_format: "NEWLINE_DELIMITED_JSON",
             ignore_unknown_values: false,
@@ -258,7 +241,7 @@ class BigQueryLoadOutputTest < Test::Unit::TestCase
         stub!.job_reference.stub!.job_id { "dummy_job_id" }
       end
 
-      mock(writer.client).get_job('yourproject_id', 'dummy_job_id') do
+      mock(writer.client).get_job('yourproject_id', 'dummy_job_id', :location=>nil) do
         stub! do |s|
           s.id { 'dummy_job_id' }
           s.configuration.stub! do |_s|
@@ -289,6 +272,61 @@ class BigQueryLoadOutputTest < Test::Unit::TestCase
     driver.instance_shutdown
   end
 
+  def test_write_with_auto_create_table
+    driver = create_driver(<<-CONFIG)
+      table foo
+      email foo@bar.example
+      private_key_path /path/to/key
+      project yourproject_id
+      dataset yourdataset_id
+
+      <buffer>
+        @type memory
+      </buffer>
+
+      <inject>
+      time_format %s
+      time_key  time
+      </inject>
+
+      auto_create_table true
+      schema_path #{SCHEMA_PATH}
+    CONFIG
+
+    schema_fields = Fluent::BigQuery::Helper.deep_symbolize_keys(MultiJson.load(File.read(SCHEMA_PATH)))
+
+    stub_writer do |writer|
+      mock(writer.client).get_table('yourproject_id', 'yourdataset_id', 'foo') do
+        raise Google::Apis::ClientError.new("notFound: Not found: Table yourproject_id:yourdataset_id.foo", status_code: 404)
+      end
+
+      mock(writer.client).insert_job('yourproject_id', {
+        configuration: {
+          load: {
+            destination_table: {
+              project_id: 'yourproject_id',
+              dataset_id: 'yourdataset_id',
+              table_id: 'foo',
+            },
+            write_disposition: "WRITE_APPEND",
+            source_format: "NEWLINE_DELIMITED_JSON",
+            ignore_unknown_values: false,
+            max_bad_records: 0,
+            schema: {
+              fields: schema_fields,
+            },
+          }
+        }
+      }, {upload_source: duck_type(:write, :sync, :rewind), content_type: "application/octet-stream"}) do
+        stub!.job_reference.stub!.job_id { "dummy_job_id" }
+      end
+    end
+
+    driver.run do
+      driver.feed("tag", Time.now.to_i, {"a" => "b"})
+    end
+  end
+
   private
 
   def create_response_stub(response)

data/test/plugin/test_record_schema.rb

@@ -27,6 +27,11 @@ class RecordSchemaTest < Test::Unit::TestCase
         "name" => "argv",
         "type" => "STRING",
         "mode" => "REPEATED"
+      },
+      {
+        "name" => "utilisation",
+        "type" => "NUMERIC",
+        "mode" => "NULLABLE"
       }
     ]
   end
@@ -58,6 +63,11 @@ class RecordSchemaTest < Test::Unit::TestCase
         "type" => "STRING",
         "mode" => "REPEATED"
       },
+      {
+        "name" => "utilisation",
+        "type" => "NUMERIC",
+        "mode" => "NULLABLE"
+      },
       {
         "name" => "new_column",
         "type" => "STRING",
@@ -93,6 +103,11 @@ class RecordSchemaTest < Test::Unit::TestCase
         "type" => "STRING",
         "mode" => "REPEATED"
       },
+      {
+        "name" => "utilisation",
+        "type" => "NUMERIC",
+        "mode" => "NULLABLE"
+      }
     ]
   end
 
@@ -142,12 +157,12 @@ class RecordSchemaTest < Test::Unit::TestCase
     time = Time.local(2016, 2, 7, 19, 0, 0).utc
 
     formatted = fields.format_one({
-      "time" => time, "tty" => ["tty1", "tty2", "tty3"], "pwd" => "/home", "user" => {name: "joker1007", uid: 10000}, "argv" => ["foo", 42]
+      "time" => time, "tty" => ["tty1", "tty2", "tty3"], "pwd" => "/home", "user" => {name: "joker1007", uid: 10000}, "argv" => ["foo", 42], "utilisation" => "0.837"
     })
     assert_equal(
       formatted,
       {
-        "time" => time.strftime("%Y-%m-%d %H:%M:%S.%6L %:z"), "tty" => MultiJson.dump(["tty1", "tty2", "tty3"]), "pwd" => "/home", "user" => MultiJson.dump({name: "joker1007", uid: 10000}), "argv" => ["foo", "42"]
+        "time" => time.strftime("%Y-%m-%d %H:%M:%S.%6L %:z"), "tty" => MultiJson.dump(["tty1", "tty2", "tty3"]), "pwd" => "/home", "user" => MultiJson.dump({name: "joker1007", uid: 10000}), "argv" => ["foo", "42"], "utilisation" => "0.837"
       }
     )
   end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-bigquery
 version: !ruby/object:Gem::Version
-  version: 2.0.0.beta
+  version: 2.3.0
 platform: ruby
 authors:
 - Naoya Ito
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-03-29 00:00:00.000000000 Z
+date: 2022-02-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -139,8 +139,9 @@ extensions: []
 extra_rdoc_files: []
 files:
 - ".github/ISSUE_TEMPLATE.md"
+- ".github/workflows/linux.yml"
+- ".github/workflows/windows.yml"
 - ".gitignore"
-- ".travis.yml"
 - Gemfile
 - LICENSE.txt
 - README.md
@@ -179,12 +180,11 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">"
+  - - ">="
     - !ruby/object:Gem::Version
-      version: 1.3.1
+      version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.6.12
+rubygems_version: 3.1.4
 signing_key: 
 specification_version: 4
 summary: Fluentd plugin to store data on Google BigQuery

data/.travis.yml

@@ -1,15 +0,0 @@
-language: ruby
-
-rvm:
-  - 2.2
-  - 2.3.3
-  - 2.4.3
-  - 2.5.0
-
-gemfile:
-  - Gemfile
-
-before_install:
-  - gem update bundler
-
-script: bundle exec rake test