logstash-output-google_bigquery 0.1.5 → 0.1.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 613b51c4536d4c8cba38acfb2be32554e12a2022
-  data.tar.gz: f78269ddde095c489da1b7e2063821225c7cb34b
+  metadata.gz: cee99e6ced7e544e287a033d2cc61b8e71187ee9
+  data.tar.gz: c3978a76beb30601a47cb445e66b72762c786c07
 SHA512:
-  metadata.gz: d1ddca6ed9f5609d16906160e919de3765a76fe2f894c8736bd93cd9ca684f2936755b6ae0eb5767c4c90d90487f375add88daca355594c44658893ec69d6cb6
-  data.tar.gz: 269c9ac785cf4c4ceab485819cce8af3a1265644d9c0adea3ebeb600d98b5b96a3f7450d6ad90e1f0a4dec59a22fd31ca38a67c8f91b486a8d2362b2076076f8
+  metadata.gz: 60b4e21d588075b540959c2fa1a3428e3e14d592336f8c9882469eee52e91838c39213476b6ab93ff6d69ecd502880d022aab8d40165c1f5b57f4ddf6be96ca8
+  data.tar.gz: ddb37b6c296ef09e3f120a4d561844f6a5b3a81eafb2574195e1a71d64a1222d6bc75cc3b590055fbe25aaaeb614173960f7cd1daedd7374af2ceb9ada707711

data/NOTICE.TXT

@@ -0,0 +1,5 @@
+Elasticsearch
+Copyright 2012-2015 Elasticsearch
+
+This product includes software developed by The Apache Software
+Foundation (http://www.apache.org/).

data/README.md

@@ -13,7 +13,7 @@ Logstash provides infrastructure to automatically generate documentation for thi
 
 ## Need Help?
 
-Need help? Try #logstash on freenode IRC or the logstash-users@googlegroups.com mailing list.
+Need help? Try #logstash on freenode IRC or the https://discuss.elastic.co/c/logstash discussion forum.
 
 ## Developing
 

data/lib/logstash/outputs/google_bigquery.rb

@@ -17,6 +17,7 @@
 # limitations under the License.
 require "logstash/outputs/base"
 require "logstash/namespace"
+require "logstash/json"
 
 # Summary: plugin to upload log events to Google BigQuery (BQ), rolling
 # files based on the date pattern provided as a configuration setting. Events
@@ -101,6 +102,10 @@ class LogStash::Outputs::GoogleBigQuery < LogStash::Outputs::Base
   # Example: path:STRING,status:INTEGER,score:FLOAT
   config :csv_schema, :validate => :string, :required => true
 
+  # Indicates if BigQuery should allow extra values that are not represented in the table schema.
+  # If true, the extra values are ignored. If false, records with extra columns are treated as bad records, and if there are too many bad records, an invalid error is returned in the job result. The default value is false.
+  config :ignore_unknown_values, :validate => :boolean, :default => false
+
   # Path to private key file for Google Service Account.
   config :key_path, :validate => :string, :required => true
 
@@ -467,7 +472,6 @@ class LogStash::Outputs::GoogleBigQuery < LogStash::Outputs::Base
   # Uploads a local file to the configured bucket.
   def get_job_status(job_id)
     begin
-      require 'json'
       @logger.debug("BQ: check job status.",
                     :job_id => job_id)
       get_result = @client.execute(:api_method => @bq.jobs.get,
@@ -475,7 +479,7 @@ class LogStash::Outputs::GoogleBigQuery < LogStash::Outputs::Base
                                      'jobId' => job_id,
                                      'projectId' => @project_id
                                    })
-      response = JSON.parse(get_result.response.body)
+      response = LogStash::Json.load(get_result.response.body)
       @logger.debug("BQ: successfully invoked API.",
                     :response => response)
 
@@ -498,7 +502,6 @@ class LogStash::Outputs::GoogleBigQuery < LogStash::Outputs::Base
   # Uploads a local file to the configured bucket.
   def upload_object(filename)
     begin
-      require 'json'
       table_id = @table_prefix + "_" + get_date_pattern(filename)
       # BQ does not accept anything other than alphanumeric and _
       # Ref: https://developers.google.com/bigquery/browser-tool-quickstart?hl=en
@@ -519,7 +522,8 @@ class LogStash::Outputs::GoogleBigQuery < LogStash::Outputs::Base
               "tableId" => table_id
             },
             'createDisposition' => 'CREATE_IF_NEEDED',
-            'writeDisposition' => 'WRITE_APPEND'
+            'writeDisposition' => 'WRITE_APPEND',
+            'ignoreUnknownValues' => @ignore_unknown_values
           }
         }
       }
@@ -531,7 +535,7 @@ class LogStash::Outputs::GoogleBigQuery < LogStash::Outputs::Base
                                       },
                                       :media => media)
 
-      job_id = JSON.parse(insert_result.response.body)["jobReference"]["jobId"]
+      job_id = LogStash::Json.load(insert_result.response.body)["jobReference"]["jobId"]
       @logger.debug("BQ: multipart insert",
                     :job_id => job_id)
       return job_id

data/logstash-output-google_bigquery.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-output-google_bigquery'
-  s.version         = '0.1.5'
+  s.version         = '0.1.6'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = "Plugin to upload log events to Google BigQuery (BQ)"
   s.description     = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program"

metadata

@@ -1,17 +1,18 @@
 --- !ruby/object:Gem::Specification
 name: logstash-output-google_bigquery
 version: !ruby/object:Gem::Version
-  version: 0.1.5
+  version: 0.1.6
 platform: ruby
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2015-04-20 00:00:00.000000000 Z
+date: 2015-07-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  requirement: !ruby/object:Gem::Requirement
+  name: logstash-core
+  version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
@@ -19,10 +20,7 @@ dependencies:
     - - <
       - !ruby/object:Gem::Version
         version: 2.0.0
-  name: logstash-core
-  prerelease: false
-  type: :runtime
-  version_requirements: !ruby/object:Gem::Requirement
+  requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
@@ -30,34 +28,36 @@ dependencies:
     - - <
       - !ruby/object:Gem::Version
         version: 2.0.0
+  prerelease: false
+  type: :runtime
 - !ruby/object:Gem::Dependency
+  name: google-api-client
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
         version: '0'
-  name: google-api-client
   prerelease: false
   type: :runtime
+- !ruby/object:Gem::Dependency
+  name: logstash-devutils
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
         version: '0'
-- !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
         version: '0'
-  name: logstash-devutils
   prerelease: false
   type: :development
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - '>='
-      - !ruby/object:Gem::Version
-        version: '0'
 description: This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program
 email: info@elastic.co
 executables: []
@@ -65,9 +65,11 @@ extensions: []
 extra_rdoc_files: []
 files:
 - .gitignore
+- CHANGELOG.md
 - CONTRIBUTORS
 - Gemfile
 - LICENSE
+- NOTICE.TXT
 - README.md
 - Rakefile
 - lib/logstash/outputs/google_bigquery.rb