logstash-output-google_bigquery 3.2.3 → 3.2.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: f3cfec81c2e6ac9cea32ee0334fbcf55bbef14f77a56be666026979437fff08d
-  data.tar.gz: 7f93b13e031ef385d7b68bc4875a57c42669ae003da5bad790a5585ed5b0bfd0
+  metadata.gz: aa0d62d4409731fe59435066046e0ab5a7696f4becef0f896bb6e86cac701ae9
+  data.tar.gz: 4a19742786e58c9a9aa97262f25a5773e155c6b31b33d9e3760025aba45ae87a
 SHA512:
-  metadata.gz: 67ab094422b83d711fc78117a38dc46d91e3beee142f47885f9492973da2fb205d9611dc92bdc8c8d9f61fafd6b961eae94ea06633c45f367f508bf60af15919
-  data.tar.gz: fa5ba368d6f11b897756ac7fde47f3f7ad88cef01fa2f169ecf09cb2f50d2b44b4cc8be0be40c199470d0669ff1e76b601d35a9afa8433a68eaa12200b8ac3e8
+  metadata.gz: 6ea1f2b8c954536561fe81aa9b8c1fcfbb06553ac455f00f45fcbc66440027f36ac57ff24bbb769a189d661922d5b9dcd0a4527115563c084e4bf2a26c80cef9
+  data.tar.gz: de8befd5d24ba28da45b33f0cbdb395450110eaec708484e49441c768f0231e793f23a63c00febae31b0e1579a5a9ac0a4f92dfa5de7512ddc7988a388a1ef3e

data/CHANGELOG.md

@@ -1,3 +1,6 @@
+## 3.2.4
+  - Docs: Set the default_codec doc attribute.
+
 ## 3.2.3
   - Update gemspec summary
 

data/LICENSE

@@ -1,4 +1,4 @@
-Copyright (c) 2012–2016 Elasticsearch <http://www.elastic.co>
+Copyright (c) 2012-2018 Elasticsearch <http://www.elastic.co>
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.

data/docs/index.asciidoc

@@ -1,5 +1,6 @@
 :plugin: google_bigquery
 :type: output
+:default_codec: plain
 
 ///////////////////////////////////////////
 START - GENERATED VARIABLES, DO NOT EDIT!
@@ -299,4 +300,6 @@ around one hour).
 
 
 [id="plugins-{type}s-{plugin}-common-options"]
-include::{include_path}/{type}.asciidoc[]
+include::{include_path}/{type}.asciidoc[]
+
+:default_codec!:

data/lib/logstash/outputs/google_bigquery.rb

@@ -129,8 +129,18 @@ class LogStash::Outputs::GoogleBigQuery < LogStash::Outputs::Base
   # }
   config :json_schema, :validate => :hash, :required => false, :default => nil
 
-  # Indicates if BigQuery should allow extra values that are not represented in the table schema.
-  # If true, the extra values are ignored. If false, records with extra columns are treated as bad records, and if there are too many bad records, an invalid error is returned in the job result. The default value is false.
+  # Indicates if BigQuery should ignore values that are not represented in the table schema.
+  # If true, the extra values are discarded.
+  # If false, BigQuery will reject the records with extra fields and the job will fail.
+  # The default value is false.
+  #
+  # NOTE: You may want to add a Logstash filter like the following to remove common fields it adds:
+  # [source,ruby]
+  # ----------------------------------
+  # mutate {
+  #     remove_field => ["@version","@timestamp","path","host","type", "message"]
+  # }
+  # ----------------------------------
   config :ignore_unknown_values, :validate => :boolean, :default => false
 
   # Path to private key file for Google Service Account.
@@ -217,6 +227,9 @@ class LogStash::Outputs::GoogleBigQuery < LogStash::Outputs::Base
   def receive(event)
     @logger.debug("BQ: receive method called", :event => event)
 
+    # TODO validate the schema if @ignore_unknown_values is off and alert the user now.
+    # consider creating a bad-data table to store invalid records
+
     # Message must be written as json
     message = LogStash::Json.dump(event.to_hash)
     # Remove "@" from property names
@@ -360,7 +373,7 @@ class LogStash::Outputs::GoogleBigQuery < LogStash::Outputs::Base
         when "DONE"
           if job_status["status"].has_key?("errorResult")
             @logger.error("BQ: job failed, please enable debug and check full "\
-                          "response (probably the issue is an incompatible "\
+                          "response (the issue is probably an incompatible "\
                           "schema). NOT deleting local file.",
                           :job_id => job_id,
                           :filename => filename,
@@ -411,9 +424,13 @@ class LogStash::Outputs::GoogleBigQuery < LogStash::Outputs::Base
               @logger.debug("BQ: reenqueue as log file is being currently appended to.",
                             :filename => filename)
               @upload_queue << filename
+
               # If we got here, it means that older files were uploaded, so let's
-              # wait another minute before checking on this file again.
-              sleep @uploader_interval_secs
+              # wait before checking on this file again.
+              #
+              # Use the min so we don't wait too long if the logs start rotating too quickly
+              # due to an increased number of events.
+              sleep [60, @uploader_interval_secs].min
               next
             else
               @logger.debug("BQ: flush and close file to be uploaded.",
@@ -435,8 +452,6 @@ class LogStash::Outputs::GoogleBigQuery < LogStash::Outputs::Base
                         :filename => filename)
           File.delete(filename)
         end
-
-        sleep @uploader_interval_secs
       end
     end
   end

data/logstash-output-google_bigquery.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name            = 'logstash-output-google_bigquery'
-  s.version         = '3.2.3'
+  s.version         = '3.2.4'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = "Writes events to Google BigQuery"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-output-google_bigquery
 version: !ruby/object:Gem::Version
-  version: 3.2.3
+  version: 3.2.4
 platform: ruby
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2017-11-07 00:00:00.000000000 Z
+date: 2018-04-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement