logstash-output-google_bigquery 2.0.3 → 2.0.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 872d8260b8414aac2be0a457bd42ce98392373bc
-  data.tar.gz: 6bca379ed332084d90d358c54b50bca7b1939419
+  metadata.gz: 34a141fe1a35f5b762787fc94ba8e33b80af5fee
+  data.tar.gz: 22c6ada4627fd4c6d8f6c02f2e1830e823caf6f9
 SHA512:
-  metadata.gz: 175bea2712feac1c5eede53a9f70e29f46abcbb176ff137d39af8cfcaa6037ae89b7157848dc261e8ba18a2bca514488f61f76b1c9dae4a6a04145f1dee92000
-  data.tar.gz: 20bf02602e61780f7446a4cfca7f1098555c9fac39a13284d7b62f279c9971b2fa225971b2366b036ad444a4cc5488479d04f960b935a52b87c3b172c02c9b66
+  metadata.gz: 2c53eb5b412d72b15a3d0195f03a1e5f27d99ff12a2004fe472f33f1c0ea3fd3996e3507e1b04eefe200d2c5d55a2048363ae69a936b19290784aba51527e033
+  data.tar.gz: ae98407e1aa2807796f374b6b0ec50ba073570ad35367085a5984242fefe0aad40745adaab803ac5bd1ef3fa4a5b33b4ef2df07c2f92fc1d6761fea7ff503ab0

data/CHANGELOG.md

@@ -1,3 +1,10 @@
+# 2.0.6
+  - Pin version of gems whose latest releases only work with ruby 2.x
+
+# 2.0.5
+  - Depend on logstash-core-plugin-api instead of logstash-core, removing the need to mass update plugins on major releases of logstash
+# 2.0.4
+  - New dependency requirements for logstash-core for the 5.0 release
 ## 2.0.3
  - Add support for specifying schema as a hash
  - Bubble up error message that BQ returns on an error

data/LICENSE

@@ -1,4 +1,4 @@
-Copyright (c) 2012–2015 Elasticsearch <http://www.elastic.co>
+Copyright (c) 2012–2016 Elasticsearch <http://www.elastic.co>
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.

data/README.md

@@ -1,7 +1,6 @@
 # Logstash Plugin
 
-[![Build
-Status](http://build-eu-00.elastic.co/view/LS%20Plugins/view/LS%20Outputs/job/logstash-plugin-output-google_bigquery-unit/badge/icon)](http://build-eu-00.elastic.co/view/LS%20Plugins/view/LS%20Outputs/job/logstash-plugin-output-google_bigquery-unit/)
+[![Travis Build Status](https://travis-ci.org/logstash-plugins/logstash-output-google_bigquery.svg)](https://travis-ci.org/logstash-plugins/logstash-output-google_bigquery)
 
 This is a plugin for [Logstash](https://github.com/elastic/logstash).
 
@@ -56,7 +55,12 @@ gem "logstash-filter-awesome", :path => "/your/local/logstash-filter-awesome"
 ```
 - Install plugin
 ```sh
+# Logstash 2.3 and higher
+bin/logstash-plugin install --no-verify
+
+# Prior to Logstash 2.3
 bin/plugin install --no-verify
+
 ```
 - Run Logstash with your plugin
 ```sh
@@ -74,7 +78,12 @@ gem build logstash-filter-awesome.gemspec
 ```
 - Install the plugin from the Logstash home
 ```sh
-bin/plugin install /your/local/plugin/logstash-filter-awesome.gem
+# Logstash 2.3 and higher
+bin/logstash-plugin install --no-verify
+
+# Prior to Logstash 2.3
+bin/plugin install --no-verify
+
 ```
 - Start Logstash and proceed to test the plugin
 

data/lib/logstash/outputs/google_bigquery.rb

@@ -96,9 +96,13 @@ class LogStash::Outputs::GoogleBigQuery < LogStash::Outputs::Base
   config :dataset, :validate => :string, :required => true
 
   # BigQuery table ID prefix to be used when creating new tables for log data.
-  # Table name will be <table_prefix>_<date>
+  # Table name will be <table_prefix><table_separator><date>
   config :table_prefix, :validate => :string, :default => "logstash"
 
+  # BigQuery table separator to be added between the table_prefix and the
+  # date suffix.
+  config :table_separator, :validate => :string, :default => "_"
+
   # Schema for log data. It must follow this format:
   # <field1-name>:<field1-type>,<field2-name>:<field2-type>,...
   # Example: path:STRING,status:INTEGER,score:FLOAT
@@ -292,12 +296,20 @@ class LogStash::Outputs::GoogleBigQuery < LogStash::Outputs::Base
   def initialize_deleter
     @uploader = Thread.new do
       @logger.debug("BQ: starting deleter")
+      Dir.glob(get_undated_path() + "*.bqjob").each do |fn|
+        job_id = File.open(fn, 'r') { |f| f.read }
+        filename = @temp_directory + File::SEPARATOR + File.basename(fn, ".bqjob")
+        @logger.debug("BQ: resuming job.",
+                      :job_id => job_id,
+                      :filename => filename)
+        @delete_queue << { "filename" => filename, "job_id" => job_id }
+      end
       while true
         delete_item = @delete_queue.pop
         job_id = delete_item["job_id"]
         filename = delete_item["filename"]
         job_status = get_job_status(job_id)
-        case job_status["state"]
+        case job_status["status"]["state"]
         when "DONE"
           if job_status.has_key?("errorResult")
             @logger.error("BQ: job failed, please enable debug and check full "\
@@ -311,7 +323,8 @@ class LogStash::Outputs::GoogleBigQuery < LogStash::Outputs::Base
                           :job_id => job_id,
                           :filename => filename,
                           :job_status => job_status)
-            File.delete(filename)
+            File.delete(filename) if File.exist?(filename)
+            File.delete(filename + ".bqjob") if File.exist?(filename + ".bqjob")
           end
         when "PENDING", "RUNNING"
           @logger.debug("BQ: job is not done, NOT deleting local file yet.",
@@ -320,12 +333,13 @@ class LogStash::Outputs::GoogleBigQuery < LogStash::Outputs::Base
                         :job_status => job_status)
           @delete_queue << delete_item
         else
-          @logger.error("BQ: unknown job status, please enable debug and "\
-                        "check full response (probably the issue is an "\
-                        "incompatible schema). NOT deleting local file yet.",
+          @logger.error("BQ: unknown job status, NOT deleting local file yet.",
                         :job_id => job_id,
                         :filename => filename,
                         :job_status => job_status)
+          File.open(filename + ".err", 'w') do |file|
+            file.write(LogStash::Json.dump(job_status))
+          end
         end
 
         sleep @deleter_interval_secs
@@ -367,6 +381,7 @@ class LogStash::Outputs::GoogleBigQuery < LogStash::Outputs::Base
         if File.size(filename) > 0
           job_id = upload_object(filename)
           @delete_queue << { "filename" => filename, "job_id" => job_id }
+          File.open(filename + ".bqjob", 'w') { |file| file.write(job_id) }
         else
           @logger.debug("BQ: skipping empty file.")
           @logger.debug("BQ: delete local temporary file ",
@@ -490,6 +505,12 @@ class LogStash::Outputs::GoogleBigQuery < LogStash::Outputs::Base
     @client.authorization = service_account.authorize
   end
 
+  def raise_if_error(response)
+    if response.has_key?("error")
+      raise response["error"]["message"]
+    end
+  end
+
   ##
   # Uploads a local file to the configured bucket.
   def get_job_status(job_id)
@@ -505,13 +526,10 @@ class LogStash::Outputs::GoogleBigQuery < LogStash::Outputs::Base
       @logger.debug("BQ: successfully invoked API.",
                     :response => response)
 
-      if response.has_key?("error")
-        raise response["error"]
-      end
+      raise_if_error(response)
 
       # Successful invocation
-      contents = response["status"]
-      return contents
+      return response
     rescue => e
       @logger.error("BQ: failed to check status", :exception => e)
       # TODO(rdc): limit retries?
@@ -524,7 +542,7 @@ class LogStash::Outputs::GoogleBigQuery < LogStash::Outputs::Base
   # Uploads a local file to the configured bucket.
   def upload_object(filename)
     begin
-      table_id = @table_prefix + "_" + get_date_pattern(filename)
+      table_id = @table_prefix + @table_separator + get_date_pattern(filename)
       # BQ does not accept anything other than alphanumeric and _
       # Ref: https://developers.google.com/bigquery/browser-tool-quickstart?hl=en
       table_id.tr!(':-','_')
@@ -557,7 +575,12 @@ class LogStash::Outputs::GoogleBigQuery < LogStash::Outputs::Base
                                       },
                                       :media => media)
 
-      job_id = LogStash::Json.load(insert_result.response.body)["jobReference"]["jobId"]
+      media.close()
+      response_body = LogStash::Json.load(insert_result.response.body)
+
+      raise_if_error(response_body)
+
+      job_id = response_body["jobReference"]["jobId"]
       @logger.debug("BQ: multipart insert",
                     :job_id => job_id)
       return job_id

data/logstash-output-google_bigquery.gemspec

@@ -1,10 +1,10 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-output-google_bigquery'
-  s.version         = '2.0.3'
+  s.version         = '2.0.6'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = "Plugin to upload log events to Google BigQuery (BQ)"
-  s.description     = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program"
+  s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
   s.authors         = ["Elastic"]
   s.email           = 'info@elastic.co'
   s.homepage        = "http://www.elastic.co/guide/en/logstash/current/index.html"
@@ -20,8 +20,8 @@ Gem::Specification.new do |s|
   s.metadata = { "logstash_plugin" => "true", "logstash_group" => "output" }
 
   # Gem dependencies
-  s.add_runtime_dependency "logstash-core", ">= 2.0.0.beta2", "< 3.0.0"
-  s.add_runtime_dependency 'google-api-client'
+  s.add_runtime_dependency "logstash-core-plugin-api", "~> 1.0"
+  s.add_runtime_dependency 'google-api-client', "~> 0.8.7" # version 0.9.x needs ruby 2.x
 
   s.add_development_dependency 'logstash-devutils'
 end

metadata

@@ -1,53 +1,47 @@
 --- !ruby/object:Gem::Specification
 name: logstash-output-google_bigquery
 version: !ruby/object:Gem::Version
-  version: 2.0.3
+  version: 2.0.6
 platform: ruby
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2016-01-04 00:00:00.000000000 Z
+date: 2016-09-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: 2.0.0.beta2
-    - - <
-      - !ruby/object:Gem::Version
-        version: 3.0.0
-  name: logstash-core
+        version: '1.0'
+  name: logstash-core-plugin-api
   prerelease: false
   type: :runtime
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
-      - !ruby/object:Gem::Version
-        version: 2.0.0.beta2
-    - - <
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: 3.0.0
+        version: '1.0'
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 0.8.7
   name: google-api-client
   prerelease: false
   type: :runtime
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 0.8.7
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   name: logstash-devutils
@@ -55,10 +49,10 @@ dependencies:
   type: :development
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-description: This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program
+description: This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program
 email: info@elastic.co
 executables: []
 extensions: []
@@ -85,17 +79,17 @@ require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
 rubyforge_project:
-rubygems_version: 2.4.5
+rubygems_version: 2.4.8
 signing_key:
 specification_version: 4
 summary: Plugin to upload log events to Google BigQuery (BQ)