logstash-output-google_cloud_storage 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,15 @@
1
+ ---
2
+ !binary "U0hBMQ==":
3
+ metadata.gz: !binary |-
4
+ ZjdiZmEwZDRmN2Q1ZThkYWQ3NGY5YjY2NjFiODRlODc1MmJhYTJhNw==
5
+ data.tar.gz: !binary |-
6
+ ZDQzZDk1MjY5N2E4Mjk1MDBkZjZkMjQ5Mjk0NjM5ODFiYTZiMjM0ZA==
7
+ SHA512:
8
+ metadata.gz: !binary |-
9
+ MjdhNWQyNWRjMjViMWExNGI1ZWI2OTFjZjdlZWMxOWYzYTk3YzUwMDk5NTNl
10
+ ZDE5N2E4ZTY2OTE4YzdlYTZkYWRmMWM0MDIzZjQyZTQwYTY2Y2M5YzVmYmZh
11
+ NzI2ZDMyNmRiM2RiOGY1MzEwM2Q4Zjk4ZmI5ZWQ3NTg0ZjE4YTg=
12
+ data.tar.gz: !binary |-
13
+ NDAxMmRiZGNkZjYxOWU0N2E5MjlkNzM0MDYxY2ZiNTI2NzFkMTgwY2QwM2I1
14
+ YjE1NDViMzNmOGY2MTJkNzhmMDc5Yjc1MWM5ZTg1N2NmNzk4ZGM0ZDVmNTk4
15
+ N2YzZTgxYmNiNTFkNWQyMTIwMWQ1MGQ4N2MxMDExN2NkYmZkZWE=
@@ -0,0 +1,4 @@
1
+ *.gem
2
+ Gemfile.lock
3
+ .bundle
4
+ vendor
data/Gemfile ADDED
@@ -0,0 +1,4 @@
1
+ source 'http://rubygems.org'
2
+ gem 'rake'
3
+ gem 'gem_publisher'
4
+ gem 'archive-tar-minitar'
data/LICENSE ADDED
@@ -0,0 +1,13 @@
1
+ Copyright (c) 2012-2014 Elasticsearch <http://www.elasticsearch.org>
2
+
3
+ Licensed under the Apache License, Version 2.0 (the "License");
4
+ you may not use this file except in compliance with the License.
5
+ You may obtain a copy of the License at
6
+
7
+ http://www.apache.org/licenses/LICENSE-2.0
8
+
9
+ Unless required by applicable law or agreed to in writing, software
10
+ distributed under the License is distributed on an "AS IS" BASIS,
11
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ See the License for the specific language governing permissions and
13
+ limitations under the License.
@@ -0,0 +1,6 @@
1
+ @files=[]
2
+
3
+ task :default do
4
+ system("rake -T")
5
+ end
6
+
@@ -0,0 +1,432 @@
1
+
2
+ # encoding: utf-8
3
+ # Author: Rodrigo De Castro <rdc@google.com>
4
+ # Date: 2013-09-20
5
+ #
6
+ # Copyright 2013 Google Inc.
7
+ #
8
+ # Licensed under the Apache License, Version 2.0 (the "License");
9
+ # you may not use this file except in compliance with the License.
10
+ # You may obtain a copy of the License at
11
+ #
12
+ # http://www.apache.org/licenses/LICENSE-2.0
13
+ #
14
+ # Unless required by applicable law or agreed to in writing, software
15
+ # distributed under the License is distributed on an "AS IS" BASIS,
16
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
17
+ # See the License for the specific language governing permissions and
18
+ # limitations under the License.
19
+ require "logstash/outputs/base"
20
+ require "logstash/namespace"
21
+ require "zlib"
22
+
23
+ # Summary: plugin to upload log events to Google Cloud Storage (GCS), rolling
24
+ # files based on the date pattern provided as a configuration setting. Events
25
+ # are written to files locally and, once file is closed, this plugin uploads
26
+ # it to the configured bucket.
27
+ #
28
+ # For more info on Google Cloud Storage, please go to:
29
+ # https://cloud.google.com/products/cloud-storage
30
+ #
31
+ # In order to use this plugin, a Google service account must be used. For
32
+ # more information, please refer to:
33
+ # https://developers.google.com/storage/docs/authentication#service_accounts
34
+ #
35
+ # Recommendation: experiment with the settings depending on how much log
36
+ # data you generate, so the uploader can keep up with the generated logs.
37
+ # Using gzip output can be a good option to reduce network traffic when
38
+ # uploading the log files and in terms of storage costs as well.
39
+ #
40
+ # USAGE:
41
+ # This is an example of logstash config:
42
+ #
43
+ # output {
44
+ # google_cloud_storage {
45
+ # bucket => "my_bucket" (required)
46
+ # key_path => "/path/to/privatekey.p12" (required)
47
+ # key_password => "notasecret" (optional)
48
+ # service_account => "1234@developer.gserviceaccount.com" (required)
49
+ # temp_directory => "/tmp/logstash-gcs" (optional)
50
+ # log_file_prefix => "logstash_gcs" (optional)
51
+ # max_file_size_kbytes => 1024 (optional)
52
+ # output_format => "plain" (optional)
53
+ # date_pattern => "%Y-%m-%dT%H:00" (optional)
54
+ # flush_interval_secs => 2 (optional)
55
+ # gzip => false (optional)
56
+ # uploader_interval_secs => 60 (optional)
57
+ # }
58
+ # }
59
+ #
60
+ # Improvements TODO list:
61
+ # - Support logstash event variables to determine filename.
62
+ # - Turn Google API code into a Plugin Mixin (like AwsConfig).
63
+ # - There's no recover method, so if logstash/plugin crashes, files may not
64
+ # be uploaded to GCS.
65
+ # - Allow user to configure file name.
66
+ # - Allow parallel uploads for heavier loads (+ connection configuration if
67
+ # exposed by Ruby API client)
68
+ class LogStash::Outputs::GoogleCloudStorage < LogStash::Outputs::Base
69
+ config_name "google_cloud_storage"
70
+ milestone 1
71
+
72
+ # GCS bucket name, without "gs://" or any other prefix.
73
+ config :bucket, :validate => :string, :required => true
74
+
75
+ # GCS path to private key file.
76
+ config :key_path, :validate => :string, :required => true
77
+
78
+ # GCS private key password.
79
+ config :key_password, :validate => :string, :default => "notasecret"
80
+
81
+ # GCS service account.
82
+ config :service_account, :validate => :string, :required => true
83
+
84
+ # Directory where temporary files are stored.
85
+ # Defaults to /tmp/logstash-gcs-<random-suffix>
86
+ config :temp_directory, :validate => :string, :default => ""
87
+
88
+ # Log file prefix. Log file will follow the format:
89
+ # <prefix>_hostname_date<.part?>.log
90
+ config :log_file_prefix, :validate => :string, :default => "logstash_gcs"
91
+
92
+ # Sets max file size in kbytes. 0 disable max file check.
93
+ config :max_file_size_kbytes, :validate => :number, :default => 10000
94
+
95
+ # The event format you want to store in files. Defaults to plain text.
96
+ config :output_format, :validate => [ "json", "plain" ], :default => "plain"
97
+
98
+ # Time pattern for log file, defaults to hourly files.
99
+ # Must Time.strftime patterns: www.ruby-doc.org/core-2.0/Time.html#method-i-strftime
100
+ config :date_pattern, :validate => :string, :default => "%Y-%m-%dT%H:00"
101
+
102
+ # Flush interval in seconds for flushing writes to log files. 0 will flush
103
+ # on every message.
104
+ config :flush_interval_secs, :validate => :number, :default => 2
105
+
106
+ # Gzip output stream when writing events to log files.
107
+ config :gzip, :validate => :boolean, :default => false
108
+
109
+ # Uploader interval when uploading new files to GCS. Adjust time based
110
+ # on your time pattern (for example, for hourly files, this interval can be
111
+ # around one hour).
112
+ config :uploader_interval_secs, :validate => :number, :default => 60
113
+
114
+ public
115
+ def register
116
+ require "fileutils"
117
+ require "thread"
118
+
119
+ @logger.debug("GCS: register plugin")
120
+
121
+ @upload_queue = Queue.new
122
+ @last_flush_cycle = Time.now
123
+ initialize_temp_directory()
124
+ initialize_current_log()
125
+ initialize_google_client()
126
+ initialize_uploader()
127
+
128
+ if @gzip
129
+ @content_type = 'application/gzip'
130
+ else
131
+ @content_type = 'text/plain'
132
+ end
133
+ end
134
+
135
+ # Method called for each log event. It writes the event to the current output
136
+ # file, flushing depending on flush interval configuration.
137
+ public
138
+ def receive(event)
139
+ return unless output?(event)
140
+
141
+ @logger.debug("GCS: receive method called", :event => event)
142
+
143
+ if (@output_format == "json")
144
+ message = event.to_json
145
+ else
146
+ message = event.to_s
147
+ end
148
+
149
+ new_base_path = get_base_path()
150
+
151
+ # Time to roll file based on the date pattern? Or is it over the size limit?
152
+ if (@current_base_path != new_base_path || (@max_file_size_kbytes > 0 && @temp_file.size >= @max_file_size_kbytes * 1024))
153
+ @logger.debug("GCS: log file will be closed and uploaded",
154
+ :filename => File.basename(@temp_file.to_path),
155
+ :size => @temp_file.size.to_s,
156
+ :max_size => @max_file_size_kbytes.to_s)
157
+ # Close does not guarantee that data is physically written to disk.
158
+ @temp_file.fsync()
159
+ @temp_file.close()
160
+ initialize_next_log()
161
+ end
162
+
163
+ @temp_file.write(message)
164
+ @temp_file.write("\n")
165
+
166
+ sync_log_file()
167
+
168
+ @logger.debug("GCS: event appended to log file",
169
+ :filename => File.basename(@temp_file.to_path))
170
+ end
171
+
172
+ public
173
+ def teardown
174
+ @logger.debug("GCS: teardown method called")
175
+
176
+ @temp_file.fsync()
177
+ @temp_file.close()
178
+ end
179
+
180
+ private
181
+ ##
182
+ # Flushes temporary log file every flush_interval_secs seconds or so.
183
+ # This is triggered by events, but if there are no events there's no point
184
+ # flushing files anyway.
185
+ #
186
+ # Inspired by lib/logstash/outputs/file.rb (flush(fd), flush_pending_files)
187
+ def sync_log_file
188
+ if flush_interval_secs <= 0
189
+ @temp_file.fsync()
190
+ return
191
+ end
192
+
193
+ return unless Time.now - @last_flush_cycle >= flush_interval_secs
194
+ @temp_file.fsync()
195
+ @logger.debug("GCS: flushing file",
196
+ :path => @temp_file.to_path,
197
+ :fd => @temp_file)
198
+ @last_flush_cycle = Time.now
199
+ end
200
+
201
+ ##
202
+ # Creates temporary directory, if it does not exist.
203
+ #
204
+ # A random suffix is appended to the temporary directory
205
+ def initialize_temp_directory
206
+ require "stud/temporary"
207
+ if @temp_directory.empty?
208
+ @temp_directory = Stud::Temporary.directory("logstash-gcs")
209
+ @logger.info("GCS: temporary directory generated",
210
+ :directory => @temp_directory)
211
+ end
212
+
213
+ if !(File.directory? @temp_directory)
214
+ @logger.debug("GCS: directory doesn't exist. Creating it.",
215
+ :directory => @temp_directory)
216
+ FileUtils.mkdir_p(@temp_directory)
217
+ end
218
+ end
219
+
220
+ ##
221
+ # Starts thread to upload log files.
222
+ #
223
+ # Uploader is done in a separate thread, not holding the receive method above.
224
+ def initialize_uploader
225
+ @uploader = Thread.new do
226
+ @logger.debug("GCS: starting uploader")
227
+ while true
228
+ filename = @upload_queue.pop
229
+
230
+ # Reenqueue if it is still the current file.
231
+ if filename == @temp_file.to_path
232
+ if @current_base_path == get_base_path()
233
+ @logger.debug("GCS: reenqueue as log file is being currently appended to.",
234
+ :filename => filename)
235
+ @upload_queue << filename
236
+ # If we got here, it means that older files were uploaded, so let's
237
+ # wait another minute before checking on this file again.
238
+ sleep @uploader_interval_secs
239
+ next
240
+ else
241
+ @logger.debug("GCS: flush and close file to be uploaded.",
242
+ :filename => filename)
243
+ @temp_file.fsync()
244
+ @temp_file.close()
245
+ initialize_next_log()
246
+ end
247
+ end
248
+
249
+ upload_object(filename)
250
+ @logger.debug("GCS: delete local temporary file ",
251
+ :filename => filename)
252
+ File.delete(filename)
253
+ sleep @uploader_interval_secs
254
+ end
255
+ end
256
+ end
257
+
258
+ ##
259
+ # Returns base path to log file that is invariant regardless of whether
260
+ # max file or gzip options.
261
+ def get_base_path
262
+ return @temp_directory + File::SEPARATOR + @log_file_prefix + "_" +
263
+ Socket.gethostname() + "_" + Time.now.strftime(@date_pattern)
264
+ end
265
+
266
+ ##
267
+ # Returns log file suffix, which will vary depending on whether gzip is
268
+ # enabled.
269
+ def get_suffix
270
+ return @gzip ? ".log.gz" : ".log"
271
+ end
272
+
273
+ ##
274
+ # Returns full path to the log file based on global variables (like
275
+ # current_base_path) and configuration options (max file size and gzip
276
+ # enabled).
277
+ def get_full_path
278
+ if @max_file_size_kbytes > 0
279
+ return @current_base_path + ".part" + ("%03d" % @size_counter) + get_suffix()
280
+ else
281
+ return @current_base_path + get_suffix()
282
+ end
283
+ end
284
+
285
+ ##
286
+ # Returns latest part number for a base path. This method checks all existing
287
+ # log files in order to find the highest part number, so this file can be used
288
+ # for appending log events.
289
+ #
290
+ # Only applicable if max file size is enabled.
291
+ def get_latest_part_number(base_path)
292
+ part_numbers = Dir.glob(base_path + ".part*" + get_suffix()).map do |item|
293
+ match = /^.*\.part(?<part_num>\d+)#{get_suffix()}$/.match(item)
294
+ next if match.nil?
295
+ match[:part_num].to_i
296
+ end
297
+
298
+ return part_numbers.max if part_numbers.any?
299
+ 0
300
+ end
301
+
302
+ ##
303
+ # Opens current log file and updates @temp_file with an instance of IOWriter.
304
+ # This method also adds file to the upload queue.
305
+ def open_current_file()
306
+ path = get_full_path()
307
+ stat = File.stat(path) rescue nil
308
+ if stat and stat.ftype == "fifo" and RUBY_PLATFORM == "java"
309
+ fd = java.io.FileWriter.new(java.io.File.new(path))
310
+ else
311
+ fd = File.new(path, "a")
312
+ end
313
+ if @gzip
314
+ fd = Zlib::GzipWriter.new(fd)
315
+ end
316
+ @temp_file = GCSIOWriter.new(fd)
317
+ @upload_queue << @temp_file.to_path
318
+ end
319
+
320
+ ##
321
+ # Opens log file on plugin initialization, trying to resume from an existing
322
+ # file. If max file size is enabled, find the highest part number and resume
323
+ # from it.
324
+ def initialize_current_log
325
+ @current_base_path = get_base_path
326
+ if @max_file_size_kbytes > 0
327
+ @size_counter = get_latest_part_number(@current_base_path)
328
+ @logger.debug("GCS: resuming from latest part.",
329
+ :part => @size_counter)
330
+ end
331
+ open_current_file()
332
+ end
333
+
334
+ ##
335
+ # Generates new log file name based on configuration options and opens log
336
+ # file. If max file size is enabled, part number if incremented in case the
337
+ # the base log file name is the same (e.g. log file was not rolled given the
338
+ # date pattern).
339
+ def initialize_next_log
340
+ new_base_path = get_base_path
341
+ if @max_file_size_kbytes > 0
342
+ @size_counter = @current_base_path == new_base_path ? @size_counter + 1 : 0
343
+ @logger.debug("GCS: opening next log file.",
344
+ :filename => @current_base_path,
345
+ :part => @size_counter)
346
+ else
347
+ @logger.debug("GCS: opening next log file.",
348
+ :filename => @current_base_path)
349
+ end
350
+ @current_base_path = new_base_path
351
+ open_current_file()
352
+ end
353
+
354
+ ##
355
+ # Initializes Google Client instantiating client and authorizing access.
356
+ def initialize_google_client
357
+ require "google/api_client"
358
+ require "openssl"
359
+
360
+ @client = Google::APIClient.new(:application_name =>
361
+ 'Logstash Google Cloud Storage output plugin',
362
+ :application_version => '0.1')
363
+ @storage = @client.discovered_api('storage', 'v1beta1')
364
+
365
+ key = Google::APIClient::PKCS12.load_key(@key_path, @key_password)
366
+ service_account = Google::APIClient::JWTAsserter.new(@service_account,
367
+ 'https://www.googleapis.com/auth/devstorage.read_write',
368
+ key)
369
+ @client.authorization = service_account.authorize
370
+ end
371
+
372
+ ##
373
+ # Uploads a local file to the configured bucket.
374
+ def upload_object(filename)
375
+ begin
376
+ @logger.debug("GCS: upload object.", :filename => filename)
377
+
378
+ media = Google::APIClient::UploadIO.new(filename, @content_type)
379
+ metadata_insert_result = @client.execute(:api_method => @storage.objects.insert,
380
+ :parameters => {
381
+ 'uploadType' => 'multipart',
382
+ 'bucket' => @bucket,
383
+ 'name' => File.basename(filename)
384
+ },
385
+ :body_object => {contentType: @content_type},
386
+ :media => media)
387
+ contents = metadata_insert_result.data
388
+ @logger.debug("GCS: multipart insert",
389
+ :object => contents.name,
390
+ :self_link => contents.self_link)
391
+ rescue => e
392
+ @logger.error("GCS: failed to upload file", :exception => e)
393
+ # TODO(rdc): limit retries?
394
+ sleep 1
395
+ retry
396
+ end
397
+ end
398
+ end
399
+
400
+ ##
401
+ # Wrapper class that abstracts which IO being used (for instance, regular
402
+ # files or GzipWriter.
403
+ #
404
+ # Inspired by lib/logstash/outputs/file.rb.
405
+ class GCSIOWriter
406
+ def initialize(io)
407
+ @io = io
408
+ end
409
+ def write(*args)
410
+ @io.write(*args)
411
+ end
412
+ def fsync
413
+ if @io.class == Zlib::GzipWriter
414
+ @io.flush
415
+ @io.to_io.fsync
416
+ else
417
+ @io.fsync
418
+ end
419
+ end
420
+ def method_missing(method_name, *args, &block)
421
+ if @io.respond_to?(method_name)
422
+ @io.send(method_name, *args, &block)
423
+ else
424
+ if @io.class == Zlib::GzipWriter && @io.to_io.respond_to?(method_name)
425
+ @io.to_io.send(method_name, *args, &block)
426
+ else
427
+ super
428
+ end
429
+ end
430
+ end
431
+ attr_accessor :active
432
+ end
@@ -0,0 +1,29 @@
1
+ Gem::Specification.new do |s|
2
+
3
+ s.name = 'logstash-output-google_cloud_storage'
4
+ s.version = '0.1.0'
5
+ s.licenses = ['Apache License (2.0)']
6
+ s.summary = "plugin to upload log events to Google Cloud Storage (GCS)"
7
+ s.description = "plugin to upload log events to Google Cloud Storage (GCS)"
8
+ s.authors = ["Elasticsearch"]
9
+ s.email = 'richard.pijnenburg@elasticsearch.com'
10
+ s.homepage = "http://logstash.net/"
11
+ s.require_paths = ["lib"]
12
+
13
+ # Files
14
+ s.files = `git ls-files`.split($\)+::Dir.glob('vendor/*')
15
+
16
+ # Tests
17
+ s.test_files = s.files.grep(%r{^(test|spec|features)/})
18
+
19
+ # Special flag to let us know this is actually a logstash plugin
20
+ s.metadata = { "logstash_plugin" => "true", "group" => "output" }
21
+
22
+ # Gem dependencies
23
+ s.add_runtime_dependency 'logstash', '>= 1.4.0', '< 2.0.0'
24
+
25
+ s.add_runtime_dependency 'stud'
26
+ s.add_runtime_dependency 'google-api-client'
27
+
28
+ end
29
+
@@ -0,0 +1,9 @@
1
+ require "gem_publisher"
2
+
3
+ desc "Publish gem to RubyGems.org"
4
+ task :publish_gem do |t|
5
+ gem_file = Dir.glob(File.expand_path('../*.gemspec',File.dirname(__FILE__))).first
6
+ gem = GemPublisher.publish_if_updated(gem_file, :rubygems)
7
+ puts "Published #{gem}" if gem
8
+ end
9
+
@@ -0,0 +1,169 @@
1
+ require "net/http"
2
+ require "uri"
3
+ require "digest/sha1"
4
+
5
+ def vendor(*args)
6
+ return File.join("vendor", *args)
7
+ end
8
+
9
+ directory "vendor/" => ["vendor"] do |task, args|
10
+ mkdir task.name
11
+ end
12
+
13
+ def fetch(url, sha1, output)
14
+
15
+ puts "Downloading #{url}"
16
+ actual_sha1 = download(url, output)
17
+
18
+ if actual_sha1 != sha1
19
+ fail "SHA1 does not match (expected '#{sha1}' but got '#{actual_sha1}')"
20
+ end
21
+ end # def fetch
22
+
23
+ def file_fetch(url, sha1)
24
+ filename = File.basename( URI(url).path )
25
+ output = "vendor/#{filename}"
26
+ task output => [ "vendor/" ] do
27
+ begin
28
+ actual_sha1 = file_sha1(output)
29
+ if actual_sha1 != sha1
30
+ fetch(url, sha1, output)
31
+ end
32
+ rescue Errno::ENOENT
33
+ fetch(url, sha1, output)
34
+ end
35
+ end.invoke
36
+
37
+ return output
38
+ end
39
+
40
+ def file_sha1(path)
41
+ digest = Digest::SHA1.new
42
+ fd = File.new(path, "r")
43
+ while true
44
+ begin
45
+ digest << fd.sysread(16384)
46
+ rescue EOFError
47
+ break
48
+ end
49
+ end
50
+ return digest.hexdigest
51
+ ensure
52
+ fd.close if fd
53
+ end
54
+
55
+ def download(url, output)
56
+ uri = URI(url)
57
+ digest = Digest::SHA1.new
58
+ tmp = "#{output}.tmp"
59
+ Net::HTTP.start(uri.host, uri.port, :use_ssl => (uri.scheme == "https")) do |http|
60
+ request = Net::HTTP::Get.new(uri.path)
61
+ http.request(request) do |response|
62
+ fail "HTTP fetch failed for #{url}. #{response}" if [200, 301].include?(response.code)
63
+ size = (response["content-length"].to_i || -1).to_f
64
+ count = 0
65
+ File.open(tmp, "w") do |fd|
66
+ response.read_body do |chunk|
67
+ fd.write(chunk)
68
+ digest << chunk
69
+ if size > 0 && $stdout.tty?
70
+ count += chunk.bytesize
71
+ $stdout.write(sprintf("\r%0.2f%%", count/size * 100))
72
+ end
73
+ end
74
+ end
75
+ $stdout.write("\r \r") if $stdout.tty?
76
+ end
77
+ end
78
+
79
+ File.rename(tmp, output)
80
+
81
+ return digest.hexdigest
82
+ rescue SocketError => e
83
+ puts "Failure while downloading #{url}: #{e}"
84
+ raise
85
+ ensure
86
+ File.unlink(tmp) if File.exist?(tmp)
87
+ end # def download
88
+
89
+ def untar(tarball, &block)
90
+ require "archive/tar/minitar"
91
+ tgz = Zlib::GzipReader.new(File.open(tarball))
92
+ # Pull out typesdb
93
+ tar = Archive::Tar::Minitar::Input.open(tgz)
94
+ tar.each do |entry|
95
+ path = block.call(entry)
96
+ next if path.nil?
97
+ parent = File.dirname(path)
98
+
99
+ mkdir_p parent unless File.directory?(parent)
100
+
101
+ # Skip this file if the output file is the same size
102
+ if entry.directory?
103
+ mkdir path unless File.directory?(path)
104
+ else
105
+ entry_mode = entry.instance_eval { @mode } & 0777
106
+ if File.exists?(path)
107
+ stat = File.stat(path)
108
+ # TODO(sissel): Submit a patch to archive-tar-minitar upstream to
109
+ # expose headers in the entry.
110
+ entry_size = entry.instance_eval { @size }
111
+ # If file sizes are same, skip writing.
112
+ next if stat.size == entry_size && (stat.mode & 0777) == entry_mode
113
+ end
114
+ puts "Extracting #{entry.full_name} from #{tarball} #{entry_mode.to_s(8)}"
115
+ File.open(path, "w") do |fd|
116
+ # eof? check lets us skip empty files. Necessary because the API provided by
117
+ # Archive::Tar::Minitar::Reader::EntryStream only mostly acts like an
118
+ # IO object. Something about empty files in this EntryStream causes
119
+ # IO.copy_stream to throw "can't convert nil into String" on JRuby
120
+ # TODO(sissel): File a bug about this.
121
+ while !entry.eof?
122
+ chunk = entry.read(16384)
123
+ fd.write(chunk)
124
+ end
125
+ #IO.copy_stream(entry, fd)
126
+ end
127
+ File.chmod(entry_mode, path)
128
+ end
129
+ end
130
+ tar.close
131
+ File.unlink(tarball) if File.file?(tarball)
132
+ end # def untar
133
+
134
+ def ungz(file)
135
+
136
+ outpath = file.gsub('.gz', '')
137
+ tgz = Zlib::GzipReader.new(File.open(file))
138
+ begin
139
+ File.open(outpath, "w") do |out|
140
+ IO::copy_stream(tgz, out)
141
+ end
142
+ File.unlink(file)
143
+ rescue
144
+ File.unlink(outpath) if File.file?(outpath)
145
+ raise
146
+ end
147
+ tgz.close
148
+ end
149
+
150
+ desc "Process any vendor files required for this plugin"
151
+ task "vendor" do |task, args|
152
+
153
+ @files.each do |file|
154
+ download = file_fetch(file['url'], file['sha1'])
155
+ if download =~ /.tar.gz/
156
+ prefix = download.gsub('.tar.gz', '').gsub('vendor/', '')
157
+ untar(download) do |entry|
158
+ if !file['files'].nil?
159
+ next unless file['files'].include?(entry.full_name.gsub(prefix, ''))
160
+ out = entry.full_name.split("/").last
161
+ end
162
+ File.join('vendor', out)
163
+ end
164
+ elsif download =~ /.gz/
165
+ ungz(download)
166
+ end
167
+ end
168
+
169
+ end
@@ -0,0 +1 @@
1
+ require 'spec_helper'
metadata ADDED
@@ -0,0 +1,103 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: logstash-output-google_cloud_storage
3
+ version: !ruby/object:Gem::Version
4
+ version: 0.1.0
5
+ platform: ruby
6
+ authors:
7
+ - Elasticsearch
8
+ autorequire:
9
+ bindir: bin
10
+ cert_chain: []
11
+ date: 2014-11-06 00:00:00.000000000 Z
12
+ dependencies:
13
+ - !ruby/object:Gem::Dependency
14
+ name: logstash
15
+ requirement: !ruby/object:Gem::Requirement
16
+ requirements:
17
+ - - ! '>='
18
+ - !ruby/object:Gem::Version
19
+ version: 1.4.0
20
+ - - <
21
+ - !ruby/object:Gem::Version
22
+ version: 2.0.0
23
+ type: :runtime
24
+ prerelease: false
25
+ version_requirements: !ruby/object:Gem::Requirement
26
+ requirements:
27
+ - - ! '>='
28
+ - !ruby/object:Gem::Version
29
+ version: 1.4.0
30
+ - - <
31
+ - !ruby/object:Gem::Version
32
+ version: 2.0.0
33
+ - !ruby/object:Gem::Dependency
34
+ name: stud
35
+ requirement: !ruby/object:Gem::Requirement
36
+ requirements:
37
+ - - ! '>='
38
+ - !ruby/object:Gem::Version
39
+ version: '0'
40
+ type: :runtime
41
+ prerelease: false
42
+ version_requirements: !ruby/object:Gem::Requirement
43
+ requirements:
44
+ - - ! '>='
45
+ - !ruby/object:Gem::Version
46
+ version: '0'
47
+ - !ruby/object:Gem::Dependency
48
+ name: google-api-client
49
+ requirement: !ruby/object:Gem::Requirement
50
+ requirements:
51
+ - - ! '>='
52
+ - !ruby/object:Gem::Version
53
+ version: '0'
54
+ type: :runtime
55
+ prerelease: false
56
+ version_requirements: !ruby/object:Gem::Requirement
57
+ requirements:
58
+ - - ! '>='
59
+ - !ruby/object:Gem::Version
60
+ version: '0'
61
+ description: plugin to upload log events to Google Cloud Storage (GCS)
62
+ email: richard.pijnenburg@elasticsearch.com
63
+ executables: []
64
+ extensions: []
65
+ extra_rdoc_files: []
66
+ files:
67
+ - .gitignore
68
+ - Gemfile
69
+ - LICENSE
70
+ - Rakefile
71
+ - lib/logstash/outputs/google_cloud_storage.rb
72
+ - logstash-output-google_cloud_storage.gemspec
73
+ - rakelib/publish.rake
74
+ - rakelib/vendor.rake
75
+ - spec/outputs/google_cloud_storage_spec.rb
76
+ homepage: http://logstash.net/
77
+ licenses:
78
+ - Apache License (2.0)
79
+ metadata:
80
+ logstash_plugin: 'true'
81
+ group: output
82
+ post_install_message:
83
+ rdoc_options: []
84
+ require_paths:
85
+ - lib
86
+ required_ruby_version: !ruby/object:Gem::Requirement
87
+ requirements:
88
+ - - ! '>='
89
+ - !ruby/object:Gem::Version
90
+ version: '0'
91
+ required_rubygems_version: !ruby/object:Gem::Requirement
92
+ requirements:
93
+ - - ! '>='
94
+ - !ruby/object:Gem::Version
95
+ version: '0'
96
+ requirements: []
97
+ rubyforge_project:
98
+ rubygems_version: 2.4.1
99
+ signing_key:
100
+ specification_version: 4
101
+ summary: plugin to upload log events to Google Cloud Storage (GCS)
102
+ test_files:
103
+ - spec/outputs/google_cloud_storage_spec.rb