logstash-output-google_cloud_storage 3.0.1 → 3.0.3
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +5 -5
- data/CHANGELOG.md +3 -0
- data/Gemfile +10 -1
- data/docs/index.asciidoc +206 -0
- data/lib/logstash/outputs/google_cloud_storage.rb +11 -5
- data/logstash-output-google_cloud_storage.gemspec +2 -2
- metadata +7 -4
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
|
-
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
2
|
+
SHA256:
|
|
3
|
+
metadata.gz: '028cf4d52779b4adfce9d03cc2046bbfb53dd0d5ff397597d8a56457811e64a4'
|
|
4
|
+
data.tar.gz: 4977802be645d4df808f5d5ded47b7ecf9628b2b1d720dadda9995eef7985d38
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 57b22937cb64c8f96b3c59285413c951407651b4910f92635958ee4bc52b0f68c4da4e9515a0c735f5637e52fea3b13d401f56fc6ffa9dad35f6b7e16873f747
|
|
7
|
+
data.tar.gz: 264829c41678e242d9986f4b3c29017a6763239a59bd74d67c6f52e40689cceb0f1709d48c185b0e30a0f88035d41567e48a2222ffa1ad04b46aba4854742eaf
|
data/CHANGELOG.md
CHANGED
data/Gemfile
CHANGED
|
@@ -1,2 +1,11 @@
|
|
|
1
1
|
source 'https://rubygems.org'
|
|
2
|
-
|
|
2
|
+
|
|
3
|
+
gemspec
|
|
4
|
+
|
|
5
|
+
logstash_path = ENV["LOGSTASH_PATH"] || "../../logstash"
|
|
6
|
+
use_logstash_source = ENV["LOGSTASH_SOURCE"] && ENV["LOGSTASH_SOURCE"].to_s == "1"
|
|
7
|
+
|
|
8
|
+
if Dir.exist?(logstash_path) && use_logstash_source
|
|
9
|
+
gem 'logstash-core', :path => "#{logstash_path}/logstash-core"
|
|
10
|
+
gem 'logstash-core-plugin-api', :path => "#{logstash_path}/logstash-core-plugin-api"
|
|
11
|
+
end
|
data/docs/index.asciidoc
ADDED
|
@@ -0,0 +1,206 @@
|
|
|
1
|
+
:plugin: google_cloud_storage
|
|
2
|
+
:type: output
|
|
3
|
+
|
|
4
|
+
///////////////////////////////////////////
|
|
5
|
+
START - GENERATED VARIABLES, DO NOT EDIT!
|
|
6
|
+
///////////////////////////////////////////
|
|
7
|
+
:version: %VERSION%
|
|
8
|
+
:release_date: %RELEASE_DATE%
|
|
9
|
+
:changelog_url: %CHANGELOG_URL%
|
|
10
|
+
:include_path: ../../../../logstash/docs/include
|
|
11
|
+
///////////////////////////////////////////
|
|
12
|
+
END - GENERATED VARIABLES, DO NOT EDIT!
|
|
13
|
+
///////////////////////////////////////////
|
|
14
|
+
|
|
15
|
+
[id="plugins-{type}-{plugin}"]
|
|
16
|
+
|
|
17
|
+
=== Google_cloud_storage output plugin
|
|
18
|
+
|
|
19
|
+
include::{include_path}/plugin_header.asciidoc[]
|
|
20
|
+
|
|
21
|
+
==== Description
|
|
22
|
+
|
|
23
|
+
Summary: plugin to upload log events to Google Cloud Storage (GCS), rolling
|
|
24
|
+
files based on the date pattern provided as a configuration setting. Events
|
|
25
|
+
are written to files locally and, once file is closed, this plugin uploads
|
|
26
|
+
it to the configured bucket.
|
|
27
|
+
|
|
28
|
+
For more info on Google Cloud Storage, please go to:
|
|
29
|
+
https://cloud.google.com/products/cloud-storage
|
|
30
|
+
|
|
31
|
+
In order to use this plugin, a Google service account must be used. For
|
|
32
|
+
more information, please refer to:
|
|
33
|
+
https://developers.google.com/storage/docs/authentication#service_accounts
|
|
34
|
+
|
|
35
|
+
Recommendation: experiment with the settings depending on how much log
|
|
36
|
+
data you generate, so the uploader can keep up with the generated logs.
|
|
37
|
+
Using gzip output can be a good option to reduce network traffic when
|
|
38
|
+
uploading the log files and in terms of storage costs as well.
|
|
39
|
+
|
|
40
|
+
USAGE:
|
|
41
|
+
This is an example of logstash config:
|
|
42
|
+
|
|
43
|
+
[source,json]
|
|
44
|
+
--------------------------
|
|
45
|
+
output {
|
|
46
|
+
google_cloud_storage {
|
|
47
|
+
bucket => "my_bucket" (required)
|
|
48
|
+
key_path => "/path/to/privatekey.p12" (required)
|
|
49
|
+
key_password => "notasecret" (optional)
|
|
50
|
+
service_account => "1234@developer.gserviceaccount.com" (required)
|
|
51
|
+
temp_directory => "/tmp/logstash-gcs" (optional)
|
|
52
|
+
log_file_prefix => "logstash_gcs" (optional)
|
|
53
|
+
max_file_size_kbytes => 1024 (optional)
|
|
54
|
+
output_format => "plain" (optional)
|
|
55
|
+
date_pattern => "%Y-%m-%dT%H:00" (optional)
|
|
56
|
+
flush_interval_secs => 2 (optional)
|
|
57
|
+
gzip => false (optional)
|
|
58
|
+
uploader_interval_secs => 60 (optional)
|
|
59
|
+
}
|
|
60
|
+
}
|
|
61
|
+
--------------------------
|
|
62
|
+
|
|
63
|
+
* Support logstash event variables to determine filename.
|
|
64
|
+
* Turn Google API code into a Plugin Mixin (like AwsConfig).
|
|
65
|
+
* There's no recover method, so if logstash/plugin crashes, files may not
|
|
66
|
+
be uploaded to GCS.
|
|
67
|
+
* Allow user to configure file name.
|
|
68
|
+
* Allow parallel uploads for heavier loads (+ connection configuration if
|
|
69
|
+
exposed by Ruby API client)
|
|
70
|
+
|
|
71
|
+
[id="plugins-{type}s-{plugin}-options"]
|
|
72
|
+
==== Google_cloud_storage Output Configuration Options
|
|
73
|
+
|
|
74
|
+
This plugin supports the following configuration options plus the <<plugins-{type}s-{plugin}-common-options>> described later.
|
|
75
|
+
|
|
76
|
+
[cols="<,<,<",options="header",]
|
|
77
|
+
|=======================================================================
|
|
78
|
+
|Setting |Input type|Required
|
|
79
|
+
| <<plugins-{type}s-{plugin}-bucket>> |<<string,string>>|Yes
|
|
80
|
+
| <<plugins-{type}s-{plugin}-date_pattern>> |<<string,string>>|No
|
|
81
|
+
| <<plugins-{type}s-{plugin}-flush_interval_secs>> |<<number,number>>|No
|
|
82
|
+
| <<plugins-{type}s-{plugin}-gzip>> |<<boolean,boolean>>|No
|
|
83
|
+
| <<plugins-{type}s-{plugin}-key_password>> |<<string,string>>|No
|
|
84
|
+
| <<plugins-{type}s-{plugin}-key_path>> |<<string,string>>|Yes
|
|
85
|
+
| <<plugins-{type}s-{plugin}-log_file_prefix>> |<<string,string>>|No
|
|
86
|
+
| <<plugins-{type}s-{plugin}-max_file_size_kbytes>> |<<number,number>>|No
|
|
87
|
+
| <<plugins-{type}s-{plugin}-output_format>> |<<string,string>>, one of `["json", "plain"]`|No
|
|
88
|
+
| <<plugins-{type}s-{plugin}-service_account>> |<<string,string>>|Yes
|
|
89
|
+
| <<plugins-{type}s-{plugin}-temp_directory>> |<<string,string>>|No
|
|
90
|
+
| <<plugins-{type}s-{plugin}-uploader_interval_secs>> |<<number,number>>|No
|
|
91
|
+
|=======================================================================
|
|
92
|
+
|
|
93
|
+
Also see <<plugins-{type}s-{plugin}-common-options>> for a list of options supported by all
|
|
94
|
+
output plugins.
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
[id="plugins-{type}s-{plugin}-bucket"]
|
|
99
|
+
===== `bucket`
|
|
100
|
+
|
|
101
|
+
* This is a required setting.
|
|
102
|
+
* Value type is <<string,string>>
|
|
103
|
+
* There is no default value for this setting.
|
|
104
|
+
|
|
105
|
+
GCS bucket name, without "gs://" or any other prefix.
|
|
106
|
+
|
|
107
|
+
[id="plugins-{type}s-{plugin}-date_pattern"]
|
|
108
|
+
===== `date_pattern`
|
|
109
|
+
|
|
110
|
+
* Value type is <<string,string>>
|
|
111
|
+
* Default value is `"%Y-%m-%dT%H:00"`
|
|
112
|
+
|
|
113
|
+
Time pattern for log file, defaults to hourly files.
|
|
114
|
+
Must Time.strftime patterns: www.ruby-doc.org/core-2.0/Time.html#method-i-strftime
|
|
115
|
+
|
|
116
|
+
[id="plugins-{type}s-{plugin}-flush_interval_secs"]
|
|
117
|
+
===== `flush_interval_secs`
|
|
118
|
+
|
|
119
|
+
* Value type is <<number,number>>
|
|
120
|
+
* Default value is `2`
|
|
121
|
+
|
|
122
|
+
Flush interval in seconds for flushing writes to log files. 0 will flush
|
|
123
|
+
on every message.
|
|
124
|
+
|
|
125
|
+
[id="plugins-{type}s-{plugin}-gzip"]
|
|
126
|
+
===== `gzip`
|
|
127
|
+
|
|
128
|
+
* Value type is <<boolean,boolean>>
|
|
129
|
+
* Default value is `false`
|
|
130
|
+
|
|
131
|
+
Gzip output stream when writing events to log files.
|
|
132
|
+
|
|
133
|
+
[id="plugins-{type}s-{plugin}-key_password"]
|
|
134
|
+
===== `key_password`
|
|
135
|
+
|
|
136
|
+
* Value type is <<string,string>>
|
|
137
|
+
* Default value is `"notasecret"`
|
|
138
|
+
|
|
139
|
+
GCS private key password.
|
|
140
|
+
|
|
141
|
+
[id="plugins-{type}s-{plugin}-key_path"]
|
|
142
|
+
===== `key_path`
|
|
143
|
+
|
|
144
|
+
* This is a required setting.
|
|
145
|
+
* Value type is <<string,string>>
|
|
146
|
+
* There is no default value for this setting.
|
|
147
|
+
|
|
148
|
+
GCS path to private key file.
|
|
149
|
+
|
|
150
|
+
[id="plugins-{type}s-{plugin}-log_file_prefix"]
|
|
151
|
+
===== `log_file_prefix`
|
|
152
|
+
|
|
153
|
+
* Value type is <<string,string>>
|
|
154
|
+
* Default value is `"logstash_gcs"`
|
|
155
|
+
|
|
156
|
+
Log file prefix. Log file will follow the format:
|
|
157
|
+
<prefix>_hostname_date<.part?>.log
|
|
158
|
+
|
|
159
|
+
[id="plugins-{type}s-{plugin}-max_file_size_kbytes"]
|
|
160
|
+
===== `max_file_size_kbytes`
|
|
161
|
+
|
|
162
|
+
* Value type is <<number,number>>
|
|
163
|
+
* Default value is `10000`
|
|
164
|
+
|
|
165
|
+
Sets max file size in kbytes. 0 disable max file check.
|
|
166
|
+
|
|
167
|
+
[id="plugins-{type}s-{plugin}-output_format"]
|
|
168
|
+
===== `output_format`
|
|
169
|
+
|
|
170
|
+
* Value can be any of: `json`, `plain`
|
|
171
|
+
* Default value is `"plain"`
|
|
172
|
+
|
|
173
|
+
The event format you want to store in files. Defaults to plain text.
|
|
174
|
+
|
|
175
|
+
[id="plugins-{type}s-{plugin}-service_account"]
|
|
176
|
+
===== `service_account`
|
|
177
|
+
|
|
178
|
+
* This is a required setting.
|
|
179
|
+
* Value type is <<string,string>>
|
|
180
|
+
* There is no default value for this setting.
|
|
181
|
+
|
|
182
|
+
GCS service account.
|
|
183
|
+
|
|
184
|
+
[id="plugins-{type}s-{plugin}-temp_directory"]
|
|
185
|
+
===== `temp_directory`
|
|
186
|
+
|
|
187
|
+
* Value type is <<string,string>>
|
|
188
|
+
* Default value is `""`
|
|
189
|
+
|
|
190
|
+
Directory where temporary files are stored.
|
|
191
|
+
Defaults to /tmp/logstash-gcs-<random-suffix>
|
|
192
|
+
|
|
193
|
+
[id="plugins-{type}s-{plugin}-uploader_interval_secs"]
|
|
194
|
+
===== `uploader_interval_secs`
|
|
195
|
+
|
|
196
|
+
* Value type is <<number,number>>
|
|
197
|
+
* Default value is `60`
|
|
198
|
+
|
|
199
|
+
Uploader interval when uploading new files to GCS. Adjust time based
|
|
200
|
+
on your time pattern (for example, for hourly files, this interval can be
|
|
201
|
+
around one hour).
|
|
202
|
+
|
|
203
|
+
|
|
204
|
+
|
|
205
|
+
[id="plugins-{type}s-{plugin}-common-options"]
|
|
206
|
+
include::{include_path}/{type}.asciidoc[]
|
|
@@ -1,3 +1,5 @@
|
|
|
1
|
+
# [source,txt]
|
|
2
|
+
# -----
|
|
1
3
|
# encoding: utf-8
|
|
2
4
|
# Author: Rodrigo De Castro <rdc@google.com>
|
|
3
5
|
# Date: 2013-09-20
|
|
@@ -15,6 +17,7 @@
|
|
|
15
17
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
16
18
|
# See the License for the specific language governing permissions and
|
|
17
19
|
# limitations under the License.
|
|
20
|
+
# -----
|
|
18
21
|
require "logstash/outputs/base"
|
|
19
22
|
require "logstash/namespace"
|
|
20
23
|
require "logstash/json"
|
|
@@ -40,6 +43,8 @@ require "zlib"
|
|
|
40
43
|
# USAGE:
|
|
41
44
|
# This is an example of logstash config:
|
|
42
45
|
#
|
|
46
|
+
# [source,json]
|
|
47
|
+
# --------------------------
|
|
43
48
|
# output {
|
|
44
49
|
# google_cloud_storage {
|
|
45
50
|
# bucket => "my_bucket" (required)
|
|
@@ -56,14 +61,15 @@ require "zlib"
|
|
|
56
61
|
# uploader_interval_secs => 60 (optional)
|
|
57
62
|
# }
|
|
58
63
|
# }
|
|
64
|
+
# --------------------------
|
|
59
65
|
#
|
|
60
66
|
# Improvements TODO list:
|
|
61
|
-
#
|
|
62
|
-
#
|
|
63
|
-
#
|
|
67
|
+
# * Support logstash event variables to determine filename.
|
|
68
|
+
# * Turn Google API code into a Plugin Mixin (like AwsConfig).
|
|
69
|
+
# * There's no recover method, so if logstash/plugin crashes, files may not
|
|
64
70
|
# be uploaded to GCS.
|
|
65
|
-
#
|
|
66
|
-
#
|
|
71
|
+
# * Allow user to configure file name.
|
|
72
|
+
# * Allow parallel uploads for heavier loads (+ connection configuration if
|
|
67
73
|
# exposed by Ruby API client)
|
|
68
74
|
class LogStash::Outputs::GoogleCloudStorage < LogStash::Outputs::Base
|
|
69
75
|
config_name "google_cloud_storage"
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Gem::Specification.new do |s|
|
|
2
2
|
s.name = 'logstash-output-google_cloud_storage'
|
|
3
|
-
s.version = '3.0.
|
|
3
|
+
s.version = '3.0.3'
|
|
4
4
|
s.licenses = ['Apache License (2.0)']
|
|
5
5
|
s.summary = "plugin to upload log events to Google Cloud Storage (GCS)"
|
|
6
6
|
s.description = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
|
|
@@ -10,7 +10,7 @@ Gem::Specification.new do |s|
|
|
|
10
10
|
s.require_paths = ["lib"]
|
|
11
11
|
|
|
12
12
|
# Files
|
|
13
|
-
s.files = Dir[
|
|
13
|
+
s.files = Dir["lib/**/*","spec/**/*","*.gemspec","*.md","CONTRIBUTORS","Gemfile","LICENSE","NOTICE.TXT", "vendor/jar-dependencies/**/*.jar", "vendor/jar-dependencies/**/*.rb", "VERSION", "docs/**/*"]
|
|
14
14
|
|
|
15
15
|
# Tests
|
|
16
16
|
s.test_files = s.files.grep(%r{^(test|spec|features)/})
|
metadata
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: logstash-output-google_cloud_storage
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 3.0.
|
|
4
|
+
version: 3.0.3
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Elastic
|
|
8
8
|
autorequire:
|
|
9
9
|
bindir: bin
|
|
10
10
|
cert_chain: []
|
|
11
|
-
date:
|
|
11
|
+
date: 2017-06-23 00:00:00.000000000 Z
|
|
12
12
|
dependencies:
|
|
13
13
|
- !ruby/object:Gem::Dependency
|
|
14
14
|
requirement: !ruby/object:Gem::Requirement
|
|
@@ -100,7 +100,9 @@ dependencies:
|
|
|
100
100
|
- - ">="
|
|
101
101
|
- !ruby/object:Gem::Version
|
|
102
102
|
version: '0'
|
|
103
|
-
description: This gem is a Logstash plugin required to be installed on top of the
|
|
103
|
+
description: This gem is a Logstash plugin required to be installed on top of the
|
|
104
|
+
Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This
|
|
105
|
+
gem is not a stand-alone program
|
|
104
106
|
email: info@elastic.co
|
|
105
107
|
executables: []
|
|
106
108
|
extensions: []
|
|
@@ -112,6 +114,7 @@ files:
|
|
|
112
114
|
- LICENSE
|
|
113
115
|
- NOTICE.TXT
|
|
114
116
|
- README.md
|
|
117
|
+
- docs/index.asciidoc
|
|
115
118
|
- lib/logstash/outputs/google_cloud_storage.rb
|
|
116
119
|
- logstash-output-google_cloud_storage.gemspec
|
|
117
120
|
- spec/outputs/google_cloud_storage_spec.rb
|
|
@@ -138,7 +141,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
|
138
141
|
version: '0'
|
|
139
142
|
requirements: []
|
|
140
143
|
rubyforge_project:
|
|
141
|
-
rubygems_version: 2.
|
|
144
|
+
rubygems_version: 2.6.11
|
|
142
145
|
signing_key:
|
|
143
146
|
specification_version: 4
|
|
144
147
|
summary: plugin to upload log events to Google Cloud Storage (GCS)
|