logstash-output-google_cloud_storage 3.2.1 → 3.3.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 86d6c651af349b7693b77ebe7483211ac98ebace
+  data.tar.gz: a760d92081ee3c5f7792b53e2d751151f141e031
+SHA512:
+  metadata.gz: 40f36c944a496412d81c0dc2346c1939db6ac171e92f17b875c8336786435cc34d8e352b849d7e27ef1fc4d57a39e51c488219b88828f1302d46380a5905f153
+  data.tar.gz: 21d8212a8a891f8e312a112bae840ef6b77791d9ada9eb0888bc5767a5579dd8c6b4140a75aeb0a6e52234f2e820c724c101476af56ce5c59eea60025d53d10c

data/CHANGELOG.md

@@ -1,3 +1,9 @@
+## 3.3.0
+Added the ability to set `gzip` as `Content-Encoding`.
+This saves storage size but still allows uncompressed downloads.
+
+- Fixes [#13](https://github.com/logstash-plugins/logstash-output-google_cloud_storage/issues/13) - Use `gzip` for `Content-Encoding` instead of `Content-Type`
+
 ## 3.2.1
   - Refactoring work to add locks to file rotation and writing.
     - Fixes [#2](https://github.com/logstash-plugins/logstash-output-google_cloud_storage/issues/2) - Plugin crashes on file rotation.

data/CONTRIBUTORS

@@ -3,7 +3,11 @@ reports, or in general have helped logstash along its way.
 
 Contributors:
 * Aaron Mildenstein (untergeek)
+* Ethan Estrada (eestrada)
+* Google LLC.
 * Jordan Sissel (jordansissel)
+* Joseph Lewis III (jlewisiii)
+* MetaPipe
 * Pier-Hugues Pellerin (ph)
 * Richard Pijnenburg (electrical)
 * Rodrigo De Castro (rdcastro)

data/docs/index.asciidoc

@@ -56,6 +56,7 @@ output {
      date_pattern => "%Y-%m-%dT%H:00"                          (optional)
      flush_interval_secs => 2                                  (optional)
      gzip => false                                             (optional)
+     gzip_content_encoding => false                            (optional)
      uploader_interval_secs => 60                              (optional)
      include_uuid => true                                      (optional)
      include_hostname => true                                  (optional)
@@ -83,6 +84,7 @@ This plugin supports the following configuration options plus the <<plugins-{typ
 | <<plugins-{type}s-{plugin}-date_pattern>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-flush_interval_secs>> |<<number,number>>|No
 | <<plugins-{type}s-{plugin}-gzip>> |<<boolean,boolean>>|No
+| <<plugins-{type}s-{plugin}-gzip_content_encoding>> |<<boolean,boolean>>|No
 | <<plugins-{type}s-{plugin}-include_hostname>> |<<boolean,boolean>>|No
 | <<plugins-{type}s-{plugin}-include_uuid>> |<<boolean,boolean>>|No
 | <<plugins-{type}s-{plugin}-key_password>> |<<string,string>>|No
@@ -134,7 +136,29 @@ on every message.
   * Value type is <<boolean,boolean>>
   * Default value is `false`
 
-Gzip output stream when writing events to log files.
+Gzip output stream when writing events to log files, set
+`Content-Type` to `application/gzip` instead of `text/plain`, and
+use file suffix `.log.gz` instead of `.log`.
+
+[id="plugins-{type}s-{plugin}-gzip_content_encoding"]
+===== `gzip_content_encoding`
+
+added[3.3.0]
+
+  * Value type is <<boolean,boolean>>
+  * Default value is `false`
+
+Gzip output stream when writing events to log files and set `Content-Encoding` to `gzip`.
+This will upload your files as `gzip` saving network and storage costs, but they will be
+transparently decompressed when you read them from the storage bucket.
+
+See the Cloud Storage documentation on https://cloud.google.com/storage/docs/metadata#content-encoding[metadata] and 
+https://cloud.google.com/storage/docs/transcoding#content-type_vs_content-encoding[transcoding]
+for more information.
+
+**Note**: It is not recommended to use both `gzip_content_encoding` and `gzip`.
+This compresses your file _twice_, will increase the work your machine does and makes 
+the files larger than just compressing once.
 
 [id="plugins-{type}s-{plugin}-include_hostname"]
 ===== `include_hostname`
@@ -245,7 +269,4 @@ around one hour).
 
 
 
-[id="plugins-{type}s-{plugin}-common-options"]
 include::{include_path}/{type}.asciidoc[]
-
-:default_codec!:

data/lib/logstash/outputs/gcs/log_rotate.rb

@@ -6,11 +6,12 @@ module LogStash
   module Outputs
     module Gcs
       class LogRotate
-        def initialize(path_factory, max_file_size_bytes, gzip, flush_interval_secs)
+        def initialize(path_factory, max_file_size_bytes, gzip, flush_interval_secs, gzip_encoded=false)
           @path_factory = path_factory
           @max_file_size_bytes = max_file_size_bytes
           @gzip = gzip
           @flush_interval_secs = flush_interval_secs
+          @gzip_encoded = gzip_encoded
 
           @lock = Concurrent::ReentrantReadWriteLock.new
           @rotate_callback = nil
@@ -47,7 +48,7 @@ module LogStash
             @path_factory.rotate_path!
 
             path = @path_factory.current_path
-            @temp_file = LogStash::Outputs::Gcs::LogFileFactory.create(path, @gzip)
+            @temp_file = LogStash::Outputs::Gcs::LogFileFactory.create(path, @gzip, true, @gzip_encoded)
           end
         end
 

data/lib/logstash/outputs/gcs/temp_log_file.rb

@@ -10,9 +10,10 @@ module LogStash
       # optionally gzipping it and creating mutexes around modification
       # points.
       class LogFileFactory
-        def self.create(path, gzip, synchronize=true)
+        def self.create(path, gzip, synchronize=true, gzip_encoded=false)
           lf = LogStash::Outputs::Gcs::PlainLogFile.new(path)
           lf = LogStash::Outputs::Gcs::GzipLogFile.new(lf) if gzip
+          lf = LogStash::Outputs::Gcs::GzipLogFile.new(lf) if gzip_encoded
           lf = LogStash::Outputs::Gcs::SynchronizedLogFile.new(lf) if synchronize
 
           lf

data/lib/logstash/outputs/google_cloud_storage.rb

@@ -63,6 +63,7 @@ require "zlib"
 #      date_pattern => "%Y-%m-%dT%H:00"                          (optional)
 #      flush_interval_secs => 2                                  (optional)
 #      gzip => false                                             (optional)
+#      gzip_content_encoding => false                            (optional)
 #      uploader_interval_secs => 60                              (optional)
 #      upload_synchronous => false                               (optional)
 #    }
@@ -114,9 +115,15 @@ class LogStash::Outputs::GoogleCloudStorage < LogStash::Outputs::Base
   # on every message.
   config :flush_interval_secs, :validate => :number, :default => 2
 
-  # Gzip output stream when writing events to log files.
+  # Gzip output stream when writing events to log files, set
+  # `Content-Type` to `application/gzip` instead of `text/plain`, and
+  # use file suffix `.log.gz` instead of `.log`.
   config :gzip, :validate => :boolean, :default => false
 
+  # Gzip output stream when writing events to log files and set
+  # `Content-Encoding` to `gzip`.
+  config :gzip_content_encoding, :validate => :boolean, :default => false
+
   # Uploader interval when uploading new files to GCS. Adjust time based
   # on your time pattern (for example, for hourly files, this interval can be
   # around one hour).
@@ -155,6 +162,7 @@ class LogStash::Outputs::GoogleCloudStorage < LogStash::Outputs::Base
     start_uploader
 
     @content_type = @gzip ? 'application/gzip' : 'text/plain'
+    @content_encoding = @gzip_content_encoding ? 'gzip' : 'identity'
   end
 
   # Method called for each log event. It writes the event to the current output
@@ -175,7 +183,7 @@ class LogStash::Outputs::GoogleCloudStorage < LogStash::Outputs::Base
   public
   def close
     @logger.debug('Stopping the plugin, uploading the remaining files.')
-    Stud.stop!(@registration_thread) unless @registration_thread.nil?
+    Thread.kill(@uploader_thread) unless @uploader_thread.nil?
 
     # Force rotate the log. If it contains data it will be submitted
     # to the work pool and will be uploaded before the plugin stops.
@@ -215,8 +223,7 @@ class LogStash::Outputs::GoogleCloudStorage < LogStash::Outputs::Base
 
   # start_uploader periodically sends flush events through the log rotater
   def start_uploader
-    Thread.new do
-      @registration_thread = Thread.current
+    @uploader_thread = Thread.new do
       Stud.interval(@uploader_interval_secs) do
         @log_rotater.writeln(nil)
       end
@@ -252,6 +259,7 @@ class LogStash::Outputs::GoogleCloudStorage < LogStash::Outputs::Base
                                                :parameters => {
                                                  'uploadType' => 'multipart',
                                                  'bucket' => @bucket,
+                                                 'contentEncoding' => @content_encoding,
                                                  'name' => File.basename(filename)
                                                },
                                                :body_object => {contentType: @content_type},
@@ -283,7 +291,7 @@ class LogStash::Outputs::GoogleCloudStorage < LogStash::Outputs::Base
 
   def initialize_log_rotater
     max_file_size = @max_file_size_kbytes * 1024
-    @log_rotater = LogStash::Outputs::Gcs::LogRotate.new(@path_factory, max_file_size, @gzip, @flush_interval_secs)
+    @log_rotater = LogStash::Outputs::Gcs::LogRotate.new(@path_factory, max_file_size, @gzip, @flush_interval_secs, @gzip_content_encoding)
 
     @log_rotater.on_rotate do |filename|
       @logger.info("Rotated out file: #{filename}")
@@ -292,4 +300,5 @@ class LogStash::Outputs::GoogleCloudStorage < LogStash::Outputs::Base
       end
     end
   end
+  attr_accessor :active
 end

data/logstash-output-google_cloud_storage.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name            = 'logstash-output-google_cloud_storage'
-  s.version         = '3.2.1'
+  s.version         = '3.3.0'
   s.licenses        = ['Apache-2.0']
   s.summary         = "plugin to upload log events to Google Cloud Storage (GCS)"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"

data/spec/outputs/gcs/temp_log_file_spec.rb

@@ -68,7 +68,7 @@ shared_examples 'a log file' do
 
   describe '#time_since_sync' do
     it 'returns a delta' do
-      expect(Time).to receive(:now).and_return(30, 40, 50)
+      expect(Time).to receive(:now).and_return(Time.at(30), Time.at(40), Time.at(50))
 
       subject.fsync
 
@@ -116,4 +116,40 @@ describe LogStash::Outputs::Gcs::SynchronizedLogFile do
   subject { LogStash::Outputs::Gcs::LogFileFactory.create(path, false, true) }
 
   it_behaves_like 'a log file'
+end
+
+describe 'gzip encoded file' do
+  let(:tempdir) { Stud::Temporary.directory }
+  let(:path) { ::File.join(tempdir, 'logfile.log') }
+  subject { LogStash::Outputs::Gcs::LogFileFactory.create(path, false, false, true) }
+
+  it_behaves_like 'a log file'
+
+  it 'creates a valid gzip' do
+    subject.write('Hello, world!')
+    subject.close!
+
+    Zlib::GzipReader.open(path) do |gz|
+      expect(gz.read).to eq('Hello, world!')
+    end
+  end
+end
+
+describe 'double gzip encoded file' do
+  let(:tempdir) { Stud::Temporary.directory }
+  let(:path) { ::File.join(tempdir, 'logfile.log') }
+  subject { LogStash::Outputs::Gcs::LogFileFactory.create(path, true, false, true) }
+
+  it_behaves_like 'a log file'
+
+  it 'creates a valid double gzip' do
+    subject.write('Hello, world!')
+    subject.close!
+
+    Zlib::GzipReader.open(path) do |outer|
+      Zlib::GzipReader.new(outer) do |inner|
+        expect(inner.read).to eq('Hello, world!')
+      end
+    end
+  end
 end

data/spec/outputs/gcs/worker_pool_spec.rb

@@ -8,6 +8,7 @@ describe LogStash::Outputs::Gcs::WorkerPool do
       expect(pool.workers).to_not receive(:post)
 
       pool.post { 1 + 2 }
+      pool.stop!
     end
 
     it 'runs the task in a different thread if asynchronous' do
@@ -15,6 +16,7 @@ describe LogStash::Outputs::Gcs::WorkerPool do
       expect(pool.workers).to receive(:post)
 
       pool.post { 1 + 2 }
+      pool.stop!
     end
 
     it 'raises an error if the pool is already stopped' do

data/spec/outputs/google_cloud_storage_spec.rb

@@ -23,5 +23,7 @@ describe LogStash::Outputs::GoogleCloudStorage do
 
   it "should register without errors" do
     expect { subject.register }.to_not raise_error
+
+    subject.close
   end
 end

metadata

@@ -1,16 +1,17 @@
 --- !ruby/object:Gem::Specification
 name: logstash-output-google_cloud_storage
 version: !ruby/object:Gem::Version
-  version: 3.2.1
+  version: 3.3.0
 platform: ruby
 authors:
 - Elastic
-autorequire:
+autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-06-05 00:00:00.000000000 Z
+date: 2018-06-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
+  name: logstash-core-plugin-api
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -19,9 +20,8 @@ dependencies:
     - - "<="
       - !ruby/object:Gem::Version
         version: '2.99'
-  name: logstash-core-plugin-api
-  prerelease: false
   type: :runtime
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -31,84 +31,84 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '2.99'
 - !ruby/object:Gem::Dependency
+  name: stud
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-  name: stud
-  prerelease: false
   type: :runtime
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
+  name: google-api-client
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
         version: 0.8.7
-  name: google-api-client
-  prerelease: false
   type: :runtime
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
         version: 0.8.7
 - !ruby/object:Gem::Dependency
+  name: logstash-codec-plain
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-  name: logstash-codec-plain
-  prerelease: false
   type: :runtime
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
+  name: mime-types
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '2'
-  name: mime-types
-  prerelease: false
   type: :runtime
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '2'
 - !ruby/object:Gem::Dependency
+  name: concurrent-ruby
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
         version: 1.0.5
-  name: concurrent-ruby
-  prerelease: false
   type: :runtime
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
         version: 1.0.5
 - !ruby/object:Gem::Dependency
+  name: logstash-devutils
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-  name: logstash-devutils
-  prerelease: false
   type: :development
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -147,7 +147,7 @@ licenses:
 metadata:
   logstash_plugin: 'true'
   logstash_group: output
-post_install_message:
+post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
@@ -162,9 +162,9 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project:
-rubygems_version: 2.6.13
-signing_key:
+rubyforge_project: 
+rubygems_version: 2.6.14
+signing_key: 
 specification_version: 4
 summary: plugin to upload log events to Google Cloud Storage (GCS)
 test_files: