logstash-output-google_cloud_storage 3.2.1 → 3.3.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/CHANGELOG.md +6 -0
- data/CONTRIBUTORS +4 -0
- data/docs/index.asciidoc +25 -4
- data/lib/logstash/outputs/gcs/log_rotate.rb +3 -2
- data/lib/logstash/outputs/gcs/temp_log_file.rb +2 -1
- data/lib/logstash/outputs/google_cloud_storage.rb +14 -5
- data/logstash-output-google_cloud_storage.gemspec +1 -1
- data/spec/outputs/gcs/temp_log_file_spec.rb +37 -1
- data/spec/outputs/gcs/worker_pool_spec.rb +2 -0
- data/spec/outputs/google_cloud_storage_spec.rb +2 -0
- metadata +21 -21
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA1:
|
3
|
+
metadata.gz: 86d6c651af349b7693b77ebe7483211ac98ebace
|
4
|
+
data.tar.gz: a760d92081ee3c5f7792b53e2d751151f141e031
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: 40f36c944a496412d81c0dc2346c1939db6ac171e92f17b875c8336786435cc34d8e352b849d7e27ef1fc4d57a39e51c488219b88828f1302d46380a5905f153
|
7
|
+
data.tar.gz: 21d8212a8a891f8e312a112bae840ef6b77791d9ada9eb0888bc5767a5579dd8c6b4140a75aeb0a6e52234f2e820c724c101476af56ce5c59eea60025d53d10c
|
data/CHANGELOG.md
CHANGED
@@ -1,3 +1,9 @@
|
|
1
|
+
## 3.3.0
|
2
|
+
Added the ability to set `gzip` as `Content-Encoding`.
|
3
|
+
This saves storage size but still allows uncompressed downloads.
|
4
|
+
|
5
|
+
- Fixes [#13](https://github.com/logstash-plugins/logstash-output-google_cloud_storage/issues/13) - Use `gzip` for `Content-Encoding` instead of `Content-Type`
|
6
|
+
|
1
7
|
## 3.2.1
|
2
8
|
- Refactoring work to add locks to file rotation and writing.
|
3
9
|
- Fixes [#2](https://github.com/logstash-plugins/logstash-output-google_cloud_storage/issues/2) - Plugin crashes on file rotation.
|
data/CONTRIBUTORS
CHANGED
@@ -3,7 +3,11 @@ reports, or in general have helped logstash along its way.
|
|
3
3
|
|
4
4
|
Contributors:
|
5
5
|
* Aaron Mildenstein (untergeek)
|
6
|
+
* Ethan Estrada (eestrada)
|
7
|
+
* Google LLC.
|
6
8
|
* Jordan Sissel (jordansissel)
|
9
|
+
* Joseph Lewis III (jlewisiii)
|
10
|
+
* MetaPipe
|
7
11
|
* Pier-Hugues Pellerin (ph)
|
8
12
|
* Richard Pijnenburg (electrical)
|
9
13
|
* Rodrigo De Castro (rdcastro)
|
data/docs/index.asciidoc
CHANGED
@@ -56,6 +56,7 @@ output {
|
|
56
56
|
date_pattern => "%Y-%m-%dT%H:00" (optional)
|
57
57
|
flush_interval_secs => 2 (optional)
|
58
58
|
gzip => false (optional)
|
59
|
+
gzip_content_encoding => false (optional)
|
59
60
|
uploader_interval_secs => 60 (optional)
|
60
61
|
include_uuid => true (optional)
|
61
62
|
include_hostname => true (optional)
|
@@ -83,6 +84,7 @@ This plugin supports the following configuration options plus the <<plugins-{typ
|
|
83
84
|
| <<plugins-{type}s-{plugin}-date_pattern>> |<<string,string>>|No
|
84
85
|
| <<plugins-{type}s-{plugin}-flush_interval_secs>> |<<number,number>>|No
|
85
86
|
| <<plugins-{type}s-{plugin}-gzip>> |<<boolean,boolean>>|No
|
87
|
+
| <<plugins-{type}s-{plugin}-gzip_content_encoding>> |<<boolean,boolean>>|No
|
86
88
|
| <<plugins-{type}s-{plugin}-include_hostname>> |<<boolean,boolean>>|No
|
87
89
|
| <<plugins-{type}s-{plugin}-include_uuid>> |<<boolean,boolean>>|No
|
88
90
|
| <<plugins-{type}s-{plugin}-key_password>> |<<string,string>>|No
|
@@ -134,7 +136,29 @@ on every message.
|
|
134
136
|
* Value type is <<boolean,boolean>>
|
135
137
|
* Default value is `false`
|
136
138
|
|
137
|
-
Gzip output stream when writing events to log files
|
139
|
+
Gzip output stream when writing events to log files, set
|
140
|
+
`Content-Type` to `application/gzip` instead of `text/plain`, and
|
141
|
+
use file suffix `.log.gz` instead of `.log`.
|
142
|
+
|
143
|
+
[id="plugins-{type}s-{plugin}-gzip_content_encoding"]
|
144
|
+
===== `gzip_content_encoding`
|
145
|
+
|
146
|
+
added[3.3.0]
|
147
|
+
|
148
|
+
* Value type is <<boolean,boolean>>
|
149
|
+
* Default value is `false`
|
150
|
+
|
151
|
+
Gzip output stream when writing events to log files and set `Content-Encoding` to `gzip`.
|
152
|
+
This will upload your files as `gzip` saving network and storage costs, but they will be
|
153
|
+
transparently decompressed when you read them from the storage bucket.
|
154
|
+
|
155
|
+
See the Cloud Storage documentation on https://cloud.google.com/storage/docs/metadata#content-encoding[metadata] and
|
156
|
+
https://cloud.google.com/storage/docs/transcoding#content-type_vs_content-encoding[transcoding]
|
157
|
+
for more information.
|
158
|
+
|
159
|
+
**Note**: It is not recommended to use both `gzip_content_encoding` and `gzip`.
|
160
|
+
This compresses your file _twice_, will increase the work your machine does and makes
|
161
|
+
the files larger than just compressing once.
|
138
162
|
|
139
163
|
[id="plugins-{type}s-{plugin}-include_hostname"]
|
140
164
|
===== `include_hostname`
|
@@ -245,7 +269,4 @@ around one hour).
|
|
245
269
|
|
246
270
|
|
247
271
|
|
248
|
-
[id="plugins-{type}s-{plugin}-common-options"]
|
249
272
|
include::{include_path}/{type}.asciidoc[]
|
250
|
-
|
251
|
-
:default_codec!:
|
@@ -6,11 +6,12 @@ module LogStash
|
|
6
6
|
module Outputs
|
7
7
|
module Gcs
|
8
8
|
class LogRotate
|
9
|
-
def initialize(path_factory, max_file_size_bytes, gzip, flush_interval_secs)
|
9
|
+
def initialize(path_factory, max_file_size_bytes, gzip, flush_interval_secs, gzip_encoded=false)
|
10
10
|
@path_factory = path_factory
|
11
11
|
@max_file_size_bytes = max_file_size_bytes
|
12
12
|
@gzip = gzip
|
13
13
|
@flush_interval_secs = flush_interval_secs
|
14
|
+
@gzip_encoded = gzip_encoded
|
14
15
|
|
15
16
|
@lock = Concurrent::ReentrantReadWriteLock.new
|
16
17
|
@rotate_callback = nil
|
@@ -47,7 +48,7 @@ module LogStash
|
|
47
48
|
@path_factory.rotate_path!
|
48
49
|
|
49
50
|
path = @path_factory.current_path
|
50
|
-
@temp_file = LogStash::Outputs::Gcs::LogFileFactory.create(path, @gzip)
|
51
|
+
@temp_file = LogStash::Outputs::Gcs::LogFileFactory.create(path, @gzip, true, @gzip_encoded)
|
51
52
|
end
|
52
53
|
end
|
53
54
|
|
@@ -10,9 +10,10 @@ module LogStash
|
|
10
10
|
# optionally gzipping it and creating mutexes around modification
|
11
11
|
# points.
|
12
12
|
class LogFileFactory
|
13
|
-
def self.create(path, gzip, synchronize=true)
|
13
|
+
def self.create(path, gzip, synchronize=true, gzip_encoded=false)
|
14
14
|
lf = LogStash::Outputs::Gcs::PlainLogFile.new(path)
|
15
15
|
lf = LogStash::Outputs::Gcs::GzipLogFile.new(lf) if gzip
|
16
|
+
lf = LogStash::Outputs::Gcs::GzipLogFile.new(lf) if gzip_encoded
|
16
17
|
lf = LogStash::Outputs::Gcs::SynchronizedLogFile.new(lf) if synchronize
|
17
18
|
|
18
19
|
lf
|
@@ -63,6 +63,7 @@ require "zlib"
|
|
63
63
|
# date_pattern => "%Y-%m-%dT%H:00" (optional)
|
64
64
|
# flush_interval_secs => 2 (optional)
|
65
65
|
# gzip => false (optional)
|
66
|
+
# gzip_content_encoding => false (optional)
|
66
67
|
# uploader_interval_secs => 60 (optional)
|
67
68
|
# upload_synchronous => false (optional)
|
68
69
|
# }
|
@@ -114,9 +115,15 @@ class LogStash::Outputs::GoogleCloudStorage < LogStash::Outputs::Base
|
|
114
115
|
# on every message.
|
115
116
|
config :flush_interval_secs, :validate => :number, :default => 2
|
116
117
|
|
117
|
-
# Gzip output stream when writing events to log files
|
118
|
+
# Gzip output stream when writing events to log files, set
|
119
|
+
# `Content-Type` to `application/gzip` instead of `text/plain`, and
|
120
|
+
# use file suffix `.log.gz` instead of `.log`.
|
118
121
|
config :gzip, :validate => :boolean, :default => false
|
119
122
|
|
123
|
+
# Gzip output stream when writing events to log files and set
|
124
|
+
# `Content-Encoding` to `gzip`.
|
125
|
+
config :gzip_content_encoding, :validate => :boolean, :default => false
|
126
|
+
|
120
127
|
# Uploader interval when uploading new files to GCS. Adjust time based
|
121
128
|
# on your time pattern (for example, for hourly files, this interval can be
|
122
129
|
# around one hour).
|
@@ -155,6 +162,7 @@ class LogStash::Outputs::GoogleCloudStorage < LogStash::Outputs::Base
|
|
155
162
|
start_uploader
|
156
163
|
|
157
164
|
@content_type = @gzip ? 'application/gzip' : 'text/plain'
|
165
|
+
@content_encoding = @gzip_content_encoding ? 'gzip' : 'identity'
|
158
166
|
end
|
159
167
|
|
160
168
|
# Method called for each log event. It writes the event to the current output
|
@@ -175,7 +183,7 @@ class LogStash::Outputs::GoogleCloudStorage < LogStash::Outputs::Base
|
|
175
183
|
public
|
176
184
|
def close
|
177
185
|
@logger.debug('Stopping the plugin, uploading the remaining files.')
|
178
|
-
|
186
|
+
Thread.kill(@uploader_thread) unless @uploader_thread.nil?
|
179
187
|
|
180
188
|
# Force rotate the log. If it contains data it will be submitted
|
181
189
|
# to the work pool and will be uploaded before the plugin stops.
|
@@ -215,8 +223,7 @@ class LogStash::Outputs::GoogleCloudStorage < LogStash::Outputs::Base
|
|
215
223
|
|
216
224
|
# start_uploader periodically sends flush events through the log rotater
|
217
225
|
def start_uploader
|
218
|
-
Thread.new do
|
219
|
-
@registration_thread = Thread.current
|
226
|
+
@uploader_thread = Thread.new do
|
220
227
|
Stud.interval(@uploader_interval_secs) do
|
221
228
|
@log_rotater.writeln(nil)
|
222
229
|
end
|
@@ -252,6 +259,7 @@ class LogStash::Outputs::GoogleCloudStorage < LogStash::Outputs::Base
|
|
252
259
|
:parameters => {
|
253
260
|
'uploadType' => 'multipart',
|
254
261
|
'bucket' => @bucket,
|
262
|
+
'contentEncoding' => @content_encoding,
|
255
263
|
'name' => File.basename(filename)
|
256
264
|
},
|
257
265
|
:body_object => {contentType: @content_type},
|
@@ -283,7 +291,7 @@ class LogStash::Outputs::GoogleCloudStorage < LogStash::Outputs::Base
|
|
283
291
|
|
284
292
|
def initialize_log_rotater
|
285
293
|
max_file_size = @max_file_size_kbytes * 1024
|
286
|
-
@log_rotater = LogStash::Outputs::Gcs::LogRotate.new(@path_factory, max_file_size, @gzip, @flush_interval_secs)
|
294
|
+
@log_rotater = LogStash::Outputs::Gcs::LogRotate.new(@path_factory, max_file_size, @gzip, @flush_interval_secs, @gzip_content_encoding)
|
287
295
|
|
288
296
|
@log_rotater.on_rotate do |filename|
|
289
297
|
@logger.info("Rotated out file: #{filename}")
|
@@ -292,4 +300,5 @@ class LogStash::Outputs::GoogleCloudStorage < LogStash::Outputs::Base
|
|
292
300
|
end
|
293
301
|
end
|
294
302
|
end
|
303
|
+
attr_accessor :active
|
295
304
|
end
|
@@ -1,6 +1,6 @@
|
|
1
1
|
Gem::Specification.new do |s|
|
2
2
|
s.name = 'logstash-output-google_cloud_storage'
|
3
|
-
s.version = '3.
|
3
|
+
s.version = '3.3.0'
|
4
4
|
s.licenses = ['Apache-2.0']
|
5
5
|
s.summary = "plugin to upload log events to Google Cloud Storage (GCS)"
|
6
6
|
s.description = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
|
@@ -68,7 +68,7 @@ shared_examples 'a log file' do
|
|
68
68
|
|
69
69
|
describe '#time_since_sync' do
|
70
70
|
it 'returns a delta' do
|
71
|
-
expect(Time).to receive(:now).and_return(30, 40, 50)
|
71
|
+
expect(Time).to receive(:now).and_return(Time.at(30), Time.at(40), Time.at(50))
|
72
72
|
|
73
73
|
subject.fsync
|
74
74
|
|
@@ -116,4 +116,40 @@ describe LogStash::Outputs::Gcs::SynchronizedLogFile do
|
|
116
116
|
subject { LogStash::Outputs::Gcs::LogFileFactory.create(path, false, true) }
|
117
117
|
|
118
118
|
it_behaves_like 'a log file'
|
119
|
+
end
|
120
|
+
|
121
|
+
describe 'gzip encoded file' do
|
122
|
+
let(:tempdir) { Stud::Temporary.directory }
|
123
|
+
let(:path) { ::File.join(tempdir, 'logfile.log') }
|
124
|
+
subject { LogStash::Outputs::Gcs::LogFileFactory.create(path, false, false, true) }
|
125
|
+
|
126
|
+
it_behaves_like 'a log file'
|
127
|
+
|
128
|
+
it 'creates a valid gzip' do
|
129
|
+
subject.write('Hello, world!')
|
130
|
+
subject.close!
|
131
|
+
|
132
|
+
Zlib::GzipReader.open(path) do |gz|
|
133
|
+
expect(gz.read).to eq('Hello, world!')
|
134
|
+
end
|
135
|
+
end
|
136
|
+
end
|
137
|
+
|
138
|
+
describe 'double gzip encoded file' do
|
139
|
+
let(:tempdir) { Stud::Temporary.directory }
|
140
|
+
let(:path) { ::File.join(tempdir, 'logfile.log') }
|
141
|
+
subject { LogStash::Outputs::Gcs::LogFileFactory.create(path, true, false, true) }
|
142
|
+
|
143
|
+
it_behaves_like 'a log file'
|
144
|
+
|
145
|
+
it 'creates a valid double gzip' do
|
146
|
+
subject.write('Hello, world!')
|
147
|
+
subject.close!
|
148
|
+
|
149
|
+
Zlib::GzipReader.open(path) do |outer|
|
150
|
+
Zlib::GzipReader.new(outer) do |inner|
|
151
|
+
expect(inner.read).to eq('Hello, world!')
|
152
|
+
end
|
153
|
+
end
|
154
|
+
end
|
119
155
|
end
|
@@ -8,6 +8,7 @@ describe LogStash::Outputs::Gcs::WorkerPool do
|
|
8
8
|
expect(pool.workers).to_not receive(:post)
|
9
9
|
|
10
10
|
pool.post { 1 + 2 }
|
11
|
+
pool.stop!
|
11
12
|
end
|
12
13
|
|
13
14
|
it 'runs the task in a different thread if asynchronous' do
|
@@ -15,6 +16,7 @@ describe LogStash::Outputs::Gcs::WorkerPool do
|
|
15
16
|
expect(pool.workers).to receive(:post)
|
16
17
|
|
17
18
|
pool.post { 1 + 2 }
|
19
|
+
pool.stop!
|
18
20
|
end
|
19
21
|
|
20
22
|
it 'raises an error if the pool is already stopped' do
|
metadata
CHANGED
@@ -1,16 +1,17 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: logstash-output-google_cloud_storage
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 3.
|
4
|
+
version: 3.3.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Elastic
|
8
|
-
autorequire:
|
8
|
+
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2018-06-
|
11
|
+
date: 2018-06-08 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
|
+
name: logstash-core-plugin-api
|
14
15
|
requirement: !ruby/object:Gem::Requirement
|
15
16
|
requirements:
|
16
17
|
- - ">="
|
@@ -19,9 +20,8 @@ dependencies:
|
|
19
20
|
- - "<="
|
20
21
|
- !ruby/object:Gem::Version
|
21
22
|
version: '2.99'
|
22
|
-
name: logstash-core-plugin-api
|
23
|
-
prerelease: false
|
24
23
|
type: :runtime
|
24
|
+
prerelease: false
|
25
25
|
version_requirements: !ruby/object:Gem::Requirement
|
26
26
|
requirements:
|
27
27
|
- - ">="
|
@@ -31,84 +31,84 @@ dependencies:
|
|
31
31
|
- !ruby/object:Gem::Version
|
32
32
|
version: '2.99'
|
33
33
|
- !ruby/object:Gem::Dependency
|
34
|
+
name: stud
|
34
35
|
requirement: !ruby/object:Gem::Requirement
|
35
36
|
requirements:
|
36
37
|
- - ">="
|
37
38
|
- !ruby/object:Gem::Version
|
38
39
|
version: '0'
|
39
|
-
name: stud
|
40
|
-
prerelease: false
|
41
40
|
type: :runtime
|
41
|
+
prerelease: false
|
42
42
|
version_requirements: !ruby/object:Gem::Requirement
|
43
43
|
requirements:
|
44
44
|
- - ">="
|
45
45
|
- !ruby/object:Gem::Version
|
46
46
|
version: '0'
|
47
47
|
- !ruby/object:Gem::Dependency
|
48
|
+
name: google-api-client
|
48
49
|
requirement: !ruby/object:Gem::Requirement
|
49
50
|
requirements:
|
50
51
|
- - "~>"
|
51
52
|
- !ruby/object:Gem::Version
|
52
53
|
version: 0.8.7
|
53
|
-
name: google-api-client
|
54
|
-
prerelease: false
|
55
54
|
type: :runtime
|
55
|
+
prerelease: false
|
56
56
|
version_requirements: !ruby/object:Gem::Requirement
|
57
57
|
requirements:
|
58
58
|
- - "~>"
|
59
59
|
- !ruby/object:Gem::Version
|
60
60
|
version: 0.8.7
|
61
61
|
- !ruby/object:Gem::Dependency
|
62
|
+
name: logstash-codec-plain
|
62
63
|
requirement: !ruby/object:Gem::Requirement
|
63
64
|
requirements:
|
64
65
|
- - ">="
|
65
66
|
- !ruby/object:Gem::Version
|
66
67
|
version: '0'
|
67
|
-
name: logstash-codec-plain
|
68
|
-
prerelease: false
|
69
68
|
type: :runtime
|
69
|
+
prerelease: false
|
70
70
|
version_requirements: !ruby/object:Gem::Requirement
|
71
71
|
requirements:
|
72
72
|
- - ">="
|
73
73
|
- !ruby/object:Gem::Version
|
74
74
|
version: '0'
|
75
75
|
- !ruby/object:Gem::Dependency
|
76
|
+
name: mime-types
|
76
77
|
requirement: !ruby/object:Gem::Requirement
|
77
78
|
requirements:
|
78
79
|
- - "~>"
|
79
80
|
- !ruby/object:Gem::Version
|
80
81
|
version: '2'
|
81
|
-
name: mime-types
|
82
|
-
prerelease: false
|
83
82
|
type: :runtime
|
83
|
+
prerelease: false
|
84
84
|
version_requirements: !ruby/object:Gem::Requirement
|
85
85
|
requirements:
|
86
86
|
- - "~>"
|
87
87
|
- !ruby/object:Gem::Version
|
88
88
|
version: '2'
|
89
89
|
- !ruby/object:Gem::Dependency
|
90
|
+
name: concurrent-ruby
|
90
91
|
requirement: !ruby/object:Gem::Requirement
|
91
92
|
requirements:
|
92
93
|
- - '='
|
93
94
|
- !ruby/object:Gem::Version
|
94
95
|
version: 1.0.5
|
95
|
-
name: concurrent-ruby
|
96
|
-
prerelease: false
|
97
96
|
type: :runtime
|
97
|
+
prerelease: false
|
98
98
|
version_requirements: !ruby/object:Gem::Requirement
|
99
99
|
requirements:
|
100
100
|
- - '='
|
101
101
|
- !ruby/object:Gem::Version
|
102
102
|
version: 1.0.5
|
103
103
|
- !ruby/object:Gem::Dependency
|
104
|
+
name: logstash-devutils
|
104
105
|
requirement: !ruby/object:Gem::Requirement
|
105
106
|
requirements:
|
106
107
|
- - ">="
|
107
108
|
- !ruby/object:Gem::Version
|
108
109
|
version: '0'
|
109
|
-
name: logstash-devutils
|
110
|
-
prerelease: false
|
111
110
|
type: :development
|
111
|
+
prerelease: false
|
112
112
|
version_requirements: !ruby/object:Gem::Requirement
|
113
113
|
requirements:
|
114
114
|
- - ">="
|
@@ -147,7 +147,7 @@ licenses:
|
|
147
147
|
metadata:
|
148
148
|
logstash_plugin: 'true'
|
149
149
|
logstash_group: output
|
150
|
-
post_install_message:
|
150
|
+
post_install_message:
|
151
151
|
rdoc_options: []
|
152
152
|
require_paths:
|
153
153
|
- lib
|
@@ -162,9 +162,9 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
162
162
|
- !ruby/object:Gem::Version
|
163
163
|
version: '0'
|
164
164
|
requirements: []
|
165
|
-
rubyforge_project:
|
166
|
-
rubygems_version: 2.6.
|
167
|
-
signing_key:
|
165
|
+
rubyforge_project:
|
166
|
+
rubygems_version: 2.6.14
|
167
|
+
signing_key:
|
168
168
|
specification_version: 4
|
169
169
|
summary: plugin to upload log events to Google Cloud Storage (GCS)
|
170
170
|
test_files:
|