logstash-output-google_bigquery 2.0.2 → 2.0.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: f2967bba0572d32904de70afea37c79a061c9d55
-  data.tar.gz: 8eef865621386cb163c4f9060224b47e3911d798
+  metadata.gz: 872d8260b8414aac2be0a457bd42ce98392373bc
+  data.tar.gz: 6bca379ed332084d90d358c54b50bca7b1939419
 SHA512:
-  metadata.gz: 5b2708e8e2bde637653d3cbc33a04b8c4600b1f2884a1ae9491c1e22017268399e51fcdcc98abf8e05c6a530ba6cd4872135f100a390d4b8f694b64bb98cd41b
-  data.tar.gz: 26e36b1c3ca8219a6fe866a45744fd7af34ab62a60eb6da07599b6c22d456041905e95159600ea38868257db51128c972e09fed18b138972935337824a0cfeb1
+  metadata.gz: 175bea2712feac1c5eede53a9f70e29f46abcbb176ff137d39af8cfcaa6037ae89b7157848dc261e8ba18a2bca514488f61f76b1c9dae4a6a04145f1dee92000
+  data.tar.gz: 20bf02602e61780f7446a4cfca7f1098555c9fac39a13284d7b62f279c9971b2fa225971b2366b036ad444a4cc5488479d04f960b935a52b87c3b172c02c9b66

data/CHANGELOG.md

@@ -1,3 +1,8 @@
+## 2.0.3
+ - Add support for specifying schema as a hash
+ - Bubble up error message that BQ returns on an error
+ - Add the table_separator option on bigquery output
+
 ## 2.0.0
  - Plugins were updated to follow the new shutdown semantic, this mainly allows Logstash to instruct input plugins to terminate gracefully, 
    instead of using Thread.raise on the plugins' threads. Ref: https://github.com/elastic/logstash/pull/3895

data/README.md

@@ -1,5 +1,8 @@
 # Logstash Plugin
 
+[![Build
+Status](http://build-eu-00.elastic.co/view/LS%20Plugins/view/LS%20Outputs/job/logstash-plugin-output-google_bigquery-unit/badge/icon)](http://build-eu-00.elastic.co/view/LS%20Plugins/view/LS%20Outputs/job/logstash-plugin-output-google_bigquery-unit/)
+
 This is a plugin for [Logstash](https://github.com/elastic/logstash).
 
 It is fully free and fully open source. The license is Apache 2.0, meaning you are pretty much free to use it however you want in whatever way.

data/lib/logstash/outputs/google_bigquery.rb

@@ -66,7 +66,7 @@ require "logstash/json"
 #    google_bigquery {
 #      project_id => "folkloric-guru-278"                        (required)
 #      dataset => "logs"                                         (required)
-#      csv_schema => "path:STRING,status:INTEGER,score:FLOAT"    (required)
+#      csv_schema => "path:STRING,status:INTEGER,score:FLOAT"    (required*)
 #      key_path => "/path/to/privatekey.p12"                     (required)
 #      key_password => "notasecret"                              (optional)
 #      service_account => "1234@developer.gserviceaccount.com"   (required)
@@ -79,6 +79,8 @@ require "logstash/json"
 #    }
 # }
 #
+# * Specify either a csv_schema or a json_schema.
+#
 # Improvements TODO list:
 # - Refactor common code between Google BQ and GCS plugins.
 # - Turn Google API code into a Plugin Mixin (like AwsConfig).
@@ -100,7 +102,22 @@ class LogStash::Outputs::GoogleBigQuery < LogStash::Outputs::Base
   # Schema for log data. It must follow this format:
   # <field1-name>:<field1-type>,<field2-name>:<field2-type>,...
   # Example: path:STRING,status:INTEGER,score:FLOAT
-  config :csv_schema, :validate => :string, :required => true
+  config :csv_schema, :validate => :string, :required => false, :default => nil
+
+  # Schema for log data, as a hash. Example:
+  # json_schema => {
+  #     fields => [{
+  #         name => "timestamp"
+  #         type => "TIMESTAMP"
+  #     }, {
+  #         name => "host"
+  #         type => "STRING"
+  #     }, {
+  #         name => "message"
+  #         type => "STRING"
+  #     }]
+  # }
+  config :json_schema, :validate => :hash, :required => false, :default => nil
 
   # Indicates if BigQuery should allow extra values that are not represented in the table schema.
   # If true, the extra values are ignored. If false, records with extra columns are treated as bad records, and if there are too many bad records, an invalid error is returned in the job result. The default value is false.
@@ -148,25 +165,30 @@ class LogStash::Outputs::GoogleBigQuery < LogStash::Outputs::Base
 
     @logger.debug("BQ: register plugin")
 
-    @fields = Array.new
+    if !@csv_schema.nil?
+      @fields = Array.new
+
+      CSV.parse(@csv_schema.gsub('\"', '""')).flatten.each do |field|
+        temp = field.strip.split(":")
 
-    CSV.parse(@csv_schema.gsub('\"', '""')).flatten.each do |field|
-      temp = field.strip.split(":")
+        # Check that the field in the schema follows the format (<name>:<value>)
+        if temp.length != 2
+          raise "BigQuery schema must follow the format <field-name>:<field-value>"
+        end
 
-      # Check that the field in the schema follows the format (<name>:<value>)
-      if temp.length != 2
-        raise "BigQuery schema must follow the format <field-name>:<field-value>"
+        @fields << { "name" => temp[0], "type" => temp[1] }
       end
 
-      @fields << { "name" => temp[0], "type" => temp[1] }
-    end
+      # Check that we have at least one field in the schema
+      if @fields.length == 0
+        raise "BigQuery schema must contain at least one field"
+      end
 
-    # Check that we have at least one field in the schema
-    if @fields.length == 0
-      raise "BigQuery schema must contain at least one field"
+      @json_schema = { "fields" => @fields }
+    end
+    if @json_schema.nil?
+      raise "Configuration must provide either json_schema or csv_schema."
     end
-
-    @json_schema = { "fields" => @fields }
 
     @upload_queue = Queue.new
     @delete_queue = Queue.new

data/logstash-output-google_bigquery.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-output-google_bigquery'
-  s.version         = '2.0.2'
+  s.version         = '2.0.3'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = "Plugin to upload log events to Google BigQuery (BQ)"
   s.description     = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-output-google_bigquery
 version: !ruby/object:Gem::Version
-  version: 2.0.2
+  version: 2.0.3
 platform: ruby
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2015-10-14 00:00:00.000000000 Z
+date: 2016-01-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -95,7 +95,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project:
-rubygems_version: 2.4.8
+rubygems_version: 2.4.5
 signing_key:
 specification_version: 4
 summary: Plugin to upload log events to Google BigQuery (BQ)