logstash-output-google_cloud_storage 3.0.1 → 3.0.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: cc96477a96f187a01300f0fac4d97d4db89229f9
-  data.tar.gz: 772f75fb6128207021822643b7312956ffac53e4
+SHA256:
+  metadata.gz: '028cf4d52779b4adfce9d03cc2046bbfb53dd0d5ff397597d8a56457811e64a4'
+  data.tar.gz: 4977802be645d4df808f5d5ded47b7ecf9628b2b1d720dadda9995eef7985d38
 SHA512:
-  metadata.gz: af5d28f9e409438df55299b705f193ffb2abb8d51141ba624792987b28c3913f1efa4f6b36973107c43deb5299602f4ec3dc1dc0d437daf65f134d0c1f653f82
-  data.tar.gz: 782ded398ca86c512d0824b0f7e897406d91bb83b8a0fc3702a4a3402a51b80dbee36f47a43c1a50b4b54d0c76ad5b78ca3f22d455ac45c443a913f1e8cc711d
+  metadata.gz: 57b22937cb64c8f96b3c59285413c951407651b4910f92635958ee4bc52b0f68c4da4e9515a0c735f5637e52fea3b13d401f56fc6ffa9dad35f6b7e16873f747
+  data.tar.gz: 264829c41678e242d9986f4b3c29017a6763239a59bd74d67c6f52e40689cceb0f1709d48c185b0e30a0f88035d41567e48a2222ffa1ad04b46aba4854742eaf

data/CHANGELOG.md

@@ -1,3 +1,6 @@
+## 3.0.2
+  - Docs: Fix doc formatting
+
 ## 3.0.1
   - align the dependency on mime-type and google-api-client with the `logstash-output-google_bigquery`
 

data/Gemfile

@@ -1,2 +1,11 @@
 source 'https://rubygems.org'
-gemspec
+
+gemspec
+
+logstash_path = ENV["LOGSTASH_PATH"] || "../../logstash"
+use_logstash_source = ENV["LOGSTASH_SOURCE"] && ENV["LOGSTASH_SOURCE"].to_s == "1"
+
+if Dir.exist?(logstash_path) && use_logstash_source
+  gem 'logstash-core', :path => "#{logstash_path}/logstash-core"
+  gem 'logstash-core-plugin-api', :path => "#{logstash_path}/logstash-core-plugin-api"
+end

data/docs/index.asciidoc

@@ -0,0 +1,206 @@
+:plugin: google_cloud_storage
+:type: output
+
+///////////////////////////////////////////
+START - GENERATED VARIABLES, DO NOT EDIT!
+///////////////////////////////////////////
+:version: %VERSION%
+:release_date: %RELEASE_DATE%
+:changelog_url: %CHANGELOG_URL%
+:include_path: ../../../../logstash/docs/include
+///////////////////////////////////////////
+END - GENERATED VARIABLES, DO NOT EDIT!
+///////////////////////////////////////////
+
+[id="plugins-{type}-{plugin}"]
+
+=== Google_cloud_storage output plugin
+
+include::{include_path}/plugin_header.asciidoc[]
+
+==== Description
+
+Summary: plugin to upload log events to Google Cloud Storage (GCS), rolling
+files based on the date pattern provided as a configuration setting. Events
+are written to files locally and, once file is closed, this plugin uploads
+it to the configured bucket.
+
+For more info on Google Cloud Storage, please go to:
+https://cloud.google.com/products/cloud-storage
+
+In order to use this plugin, a Google service account must be used. For
+more information, please refer to:
+https://developers.google.com/storage/docs/authentication#service_accounts
+
+Recommendation: experiment with the settings depending on how much log
+data you generate, so the uploader can keep up with the generated logs.
+Using gzip output can be a good option to reduce network traffic when
+uploading the log files and in terms of storage costs as well.
+
+USAGE:
+This is an example of logstash config:
+
+[source,json]
+--------------------------
+output {
+   google_cloud_storage {
+     bucket => "my_bucket"                                     (required)
+     key_path => "/path/to/privatekey.p12"                     (required)
+     key_password => "notasecret"                              (optional)
+     service_account => "1234@developer.gserviceaccount.com"   (required)
+     temp_directory => "/tmp/logstash-gcs"                     (optional)
+     log_file_prefix => "logstash_gcs"                         (optional)
+     max_file_size_kbytes => 1024                              (optional)
+     output_format => "plain"                                  (optional)
+     date_pattern => "%Y-%m-%dT%H:00"                          (optional)
+     flush_interval_secs => 2                                  (optional)
+     gzip => false                                             (optional)
+     uploader_interval_secs => 60                              (optional)
+   }
+}
+--------------------------
+
+* Support logstash event variables to determine filename.
+* Turn Google API code into a Plugin Mixin (like AwsConfig).
+* There's no recover method, so if logstash/plugin crashes, files may not
+be uploaded to GCS.
+* Allow user to configure file name.
+* Allow parallel uploads for heavier loads (+ connection configuration if
+exposed by Ruby API client)
+
+[id="plugins-{type}s-{plugin}-options"]
+==== Google_cloud_storage Output Configuration Options
+
+This plugin supports the following configuration options plus the <<plugins-{type}s-{plugin}-common-options>> described later.
+
+[cols="<,<,<",options="header",]
+|=======================================================================
+|Setting |Input type|Required
+| <<plugins-{type}s-{plugin}-bucket>> |<<string,string>>|Yes
+| <<plugins-{type}s-{plugin}-date_pattern>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-flush_interval_secs>> |<<number,number>>|No
+| <<plugins-{type}s-{plugin}-gzip>> |<<boolean,boolean>>|No
+| <<plugins-{type}s-{plugin}-key_password>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-key_path>> |<<string,string>>|Yes
+| <<plugins-{type}s-{plugin}-log_file_prefix>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-max_file_size_kbytes>> |<<number,number>>|No
+| <<plugins-{type}s-{plugin}-output_format>> |<<string,string>>, one of `["json", "plain"]`|No
+| <<plugins-{type}s-{plugin}-service_account>> |<<string,string>>|Yes
+| <<plugins-{type}s-{plugin}-temp_directory>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-uploader_interval_secs>> |<<number,number>>|No
+|=======================================================================
+
+Also see <<plugins-{type}s-{plugin}-common-options>> for a list of options supported by all
+output plugins.
+
+&nbsp;
+
+[id="plugins-{type}s-{plugin}-bucket"]
+===== `bucket` 
+
+  * This is a required setting.
+  * Value type is <<string,string>>
+  * There is no default value for this setting.
+
+GCS bucket name, without "gs://" or any other prefix.
+
+[id="plugins-{type}s-{plugin}-date_pattern"]
+===== `date_pattern` 
+
+  * Value type is <<string,string>>
+  * Default value is `"%Y-%m-%dT%H:00"`
+
+Time pattern for log file, defaults to hourly files.
+Must Time.strftime patterns: www.ruby-doc.org/core-2.0/Time.html#method-i-strftime
+
+[id="plugins-{type}s-{plugin}-flush_interval_secs"]
+===== `flush_interval_secs` 
+
+  * Value type is <<number,number>>
+  * Default value is `2`
+
+Flush interval in seconds for flushing writes to log files. 0 will flush
+on every message.
+
+[id="plugins-{type}s-{plugin}-gzip"]
+===== `gzip` 
+
+  * Value type is <<boolean,boolean>>
+  * Default value is `false`
+
+Gzip output stream when writing events to log files.
+
+[id="plugins-{type}s-{plugin}-key_password"]
+===== `key_password` 
+
+  * Value type is <<string,string>>
+  * Default value is `"notasecret"`
+
+GCS private key password.
+
+[id="plugins-{type}s-{plugin}-key_path"]
+===== `key_path` 
+
+  * This is a required setting.
+  * Value type is <<string,string>>
+  * There is no default value for this setting.
+
+GCS path to private key file.
+
+[id="plugins-{type}s-{plugin}-log_file_prefix"]
+===== `log_file_prefix` 
+
+  * Value type is <<string,string>>
+  * Default value is `"logstash_gcs"`
+
+Log file prefix. Log file will follow the format:
+<prefix>_hostname_date<.part?>.log
+
+[id="plugins-{type}s-{plugin}-max_file_size_kbytes"]
+===== `max_file_size_kbytes` 
+
+  * Value type is <<number,number>>
+  * Default value is `10000`
+
+Sets max file size in kbytes. 0 disable max file check.
+
+[id="plugins-{type}s-{plugin}-output_format"]
+===== `output_format` 
+
+  * Value can be any of: `json`, `plain`
+  * Default value is `"plain"`
+
+The event format you want to store in files. Defaults to plain text.
+
+[id="plugins-{type}s-{plugin}-service_account"]
+===== `service_account` 
+
+  * This is a required setting.
+  * Value type is <<string,string>>
+  * There is no default value for this setting.
+
+GCS service account.
+
+[id="plugins-{type}s-{plugin}-temp_directory"]
+===== `temp_directory` 
+
+  * Value type is <<string,string>>
+  * Default value is `""`
+
+Directory where temporary files are stored.
+Defaults to /tmp/logstash-gcs-<random-suffix>
+
+[id="plugins-{type}s-{plugin}-uploader_interval_secs"]
+===== `uploader_interval_secs` 
+
+  * Value type is <<number,number>>
+  * Default value is `60`
+
+Uploader interval when uploading new files to GCS. Adjust time based
+on your time pattern (for example, for hourly files, this interval can be
+around one hour).
+
+
+
+[id="plugins-{type}s-{plugin}-common-options"]
+include::{include_path}/{type}.asciidoc[]

data/lib/logstash/outputs/google_cloud_storage.rb

@@ -1,3 +1,5 @@
+# [source,txt]
+# -----
 # encoding: utf-8
 # Author: Rodrigo De Castro <rdc@google.com>
 # Date: 2013-09-20
@@ -15,6 +17,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+# -----
 require "logstash/outputs/base"
 require "logstash/namespace"
 require "logstash/json"
@@ -40,6 +43,8 @@ require "zlib"
 # USAGE:
 # This is an example of logstash config:
 #
+# [source,json]
+# --------------------------
 # output {
 #    google_cloud_storage {
 #      bucket => "my_bucket"                                     (required)
@@ -56,14 +61,15 @@ require "zlib"
 #      uploader_interval_secs => 60                              (optional)
 #    }
 # }
+# --------------------------
 #
 # Improvements TODO list:
-# - Support logstash event variables to determine filename.
-# - Turn Google API code into a Plugin Mixin (like AwsConfig).
-# - There's no recover method, so if logstash/plugin crashes, files may not
+# * Support logstash event variables to determine filename.
+# * Turn Google API code into a Plugin Mixin (like AwsConfig).
+# * There's no recover method, so if logstash/plugin crashes, files may not
 # be uploaded to GCS.
-# - Allow user to configure file name.
-# - Allow parallel uploads for heavier loads (+ connection configuration if
+# * Allow user to configure file name.
+# * Allow parallel uploads for heavier loads (+ connection configuration if
 # exposed by Ruby API client)
 class LogStash::Outputs::GoogleCloudStorage < LogStash::Outputs::Base
   config_name "google_cloud_storage"

data/logstash-output-google_cloud_storage.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name            = 'logstash-output-google_cloud_storage'
-  s.version         = '3.0.1'
+  s.version         = '3.0.3'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = "plugin to upload log events to Google Cloud Storage (GCS)"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
@@ -10,7 +10,7 @@ Gem::Specification.new do |s|
   s.require_paths = ["lib"]
 
   # Files
-  s.files = Dir['lib/**/*','spec/**/*','vendor/**/*','*.gemspec','*.md','CONTRIBUTORS','Gemfile','LICENSE','NOTICE.TXT']
+  s.files = Dir["lib/**/*","spec/**/*","*.gemspec","*.md","CONTRIBUTORS","Gemfile","LICENSE","NOTICE.TXT", "vendor/jar-dependencies/**/*.jar", "vendor/jar-dependencies/**/*.rb", "VERSION", "docs/**/*"]
 
   # Tests
   s.test_files = s.files.grep(%r{^(test|spec|features)/})

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-output-google_cloud_storage
 version: !ruby/object:Gem::Version
-  version: 3.0.1
+  version: 3.0.3
 platform: ruby
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2016-09-20 00:00:00.000000000 Z
+date: 2017-06-23 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -100,7 +100,9 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-description: This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program
+description: This gem is a Logstash plugin required to be installed on top of the
+  Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This
+  gem is not a stand-alone program
 email: info@elastic.co
 executables: []
 extensions: []
@@ -112,6 +114,7 @@ files:
 - LICENSE
 - NOTICE.TXT
 - README.md
+- docs/index.asciidoc
 - lib/logstash/outputs/google_cloud_storage.rb
 - logstash-output-google_cloud_storage.gemspec
 - spec/outputs/google_cloud_storage_spec.rb
@@ -138,7 +141,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project:
-rubygems_version: 2.4.8
+rubygems_version: 2.6.11
 signing_key:
 specification_version: 4
 summary: plugin to upload log events to Google Cloud Storage (GCS)