logstash-output-google_cloud_storage 4.0.1-java → 4.1.0-java

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 4aa7c73beda983444233b7943c788a49c2a9a7ac56218e78dce4bbe1932fb39a
-  data.tar.gz: 7f88b0005ff2fada77f84d55365693e3a9eb582178923bdf240fe66db3f15ef0
+  metadata.gz: c96bc2ab86ede5dcbcb77ba9ca521f952122cb6fe2b010e3858b45fe70a2bf83
+  data.tar.gz: 44f8de48f9d8e1d225ed42a4bb30e2b38448cfee556d1491970cb4fa28fc24f6
 SHA512:
-  metadata.gz: dddd13b7892f549aa748efcfd4a8bc16f108bb840c5179006ce12b57f027d55cafc5ed18ad36c03d7c2e2a370460ed1ebec26252323096cda3407eda5158dd94
-  data.tar.gz: 2195cf539ad95c824a0c1f4f93527c76c6ff9c0ae5c6b9c98d44670004372ef49bd5fdfc843461e3d6879f11818a5bb5ad5103efde785193484ea4b73dcc07a9
+  metadata.gz: c198cf37eaf81b51e3325af6b519782780fdf0334a81b78654df6663f11c3eda7466297bc997b5dd1551f97cf4f667791bf54d2292aeba3a87d6f672f22c43a3
+  data.tar.gz: 9a872177cfd7b33cc934983f708414d6cd06abfd1f2d2ab0d69e576403bcab0ac67d705197cce9fc266e5239937b7e0532bf4613c47d2b9de8cf18b7d92a8714

data/CHANGELOG.md

@@ -1,3 +1,6 @@
+## 4.1.0
+  - Added ability to use Logstash codecs.
+
 ## 4.0.1
   - Fixes [#38](https://github.com/logstash-plugins/logstash-output-google_cloud_storage/issues/38) - Plugin doesn't start on logstash 7.1.1 - TypeError
 

data/docs/index.asciidoc

@@ -1,6 +1,6 @@
 :plugin: google_cloud_storage
 :type: output
-:default_codec: plain
+:default_codec: line
 
 ///////////////////////////////////////////
 START - GENERATED VARIABLES, DO NOT EDIT!
@@ -101,7 +101,7 @@ This plugin supports the following configuration options plus the <<plugins-{typ
 | <<plugins-{type}s-{plugin}-log_file_prefix>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-max_concurrent_uploads>> |<<number,number>>|No
 | <<plugins-{type}s-{plugin}-max_file_size_kbytes>> |<<number,number>>|No
-| <<plugins-{type}s-{plugin}-output_format>> |<<string,string>>, one of `["json", "plain"]`|No
+| <<plugins-{type}s-{plugin}-output_format>> |<<string,string>>, one of `["json", "plain", nil]`|__Deprecated__
 | <<plugins-{type}s-{plugin}-service_account>> |<<string,string>>|__Deprecated__
 | <<plugins-{type}s-{plugin}-temp_directory>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-uploader_interval_secs>> |<<number,number>>|No
@@ -264,11 +264,18 @@ Sets max file size in kbytes. 0 disable max file check.
 [id="plugins-{type}s-{plugin}-output_format"]
 ===== `output_format` 
 
-  * Value can be any of: `json`, `plain`
-  * Default value is `"plain"`
+  * Value can be any of: `json`, `plain`, or no value
+  * Default value is no value
+
+**Deprecated**, this feature will be removed in the next major release. Use codecs instead.
+
+  * If you are using the `json` value today, switch to the `json_lines` codec.
+  * If you are using the `plain` value today, switch to the `line` codec.
 
 The event format you want to store in files. Defaults to plain text.
 
+Note: if you want to use a codec you MUST not set this value.
+
 [id="plugins-{type}s-{plugin}-service_account"]
 ===== `service_account` 
 

data/lib/logstash/outputs/gcs/log_rotate.rb

@@ -19,15 +19,15 @@ module LogStash
           rotate_log!
         end
 
-        # writeln writes a message and carriage-return character to the open
-        # log file, rotating and syncing it if necessary.
+        # write writes zero or more messages to the open log file
+        # rotating and syncing it if necessary.
         #
         # nil messages do not get written, but may cause the log to rotate
-        def writeln(message=nil)
+        def write(*messages)
           @lock.with_write_lock do
             rotate_log! if should_rotate?
 
-            @temp_file.write(message, "\n") unless message.nil?
+            @temp_file.write(*messages) unless messages.empty?
 
             @temp_file.fsync if @temp_file.time_since_sync >= @flush_interval_secs
           end

data/lib/logstash/outputs/google_cloud_storage.rb

@@ -58,7 +58,6 @@ require "zlib"
 #      temp_directory => "/tmp/logstash-gcs"                     (optional)
 #      log_file_prefix => "logstash_gcs"                         (optional)
 #      max_file_size_kbytes => 1024                              (optional)
-#      output_format => "plain"                                  (optional)
 #      date_pattern => "%Y-%m-%dT%H:00"                          (optional)
 #      flush_interval_secs => 2                                  (optional)
 #      gzip => false                                             (optional)
@@ -79,6 +78,7 @@ class LogStash::Outputs::GoogleCloudStorage < LogStash::Outputs::Base
   config_name "google_cloud_storage"
 
   concurrency :single
+  default :codec, "line"
 
   # GCS bucket name, without "gs://" or any other prefix.
   config :bucket, :validate => :string, :required => true
@@ -104,7 +104,7 @@ class LogStash::Outputs::GoogleCloudStorage < LogStash::Outputs::Base
   config :max_file_size_kbytes, :validate => :number, :default => 10000
 
   # The event format you want to store in files. Defaults to plain text.
-  config :output_format, :validate => [ "json", "plain" ], :default => "plain"
+  config :output_format, :validate => [ "json", "plain", nil ], :default => nil, :deprecated => 'Use codec instead.'
 
   # Time pattern for log file, defaults to hourly files.
   # Must Time.strftime patterns: www.ruby-doc.org/core-2.0/Time.html#method-i-strftime
@@ -154,10 +154,18 @@ class LogStash::Outputs::GoogleCloudStorage < LogStash::Outputs::Base
 
   config :max_concurrent_uploads, :validate  => :number, :default => 5
 
+  attr_accessor :disable_uploader
+
   public
   def register
     @logger.debug('Registering Google Cloud Storage plugin')
 
+    # NOTE: this is a hacky solution to get around the fact that we used to
+    # do our own pseudo-codec processing. This should be removed in the
+    # next major release.
+    params['codec'] = LogStash::Plugin.lookup('codec', 'json_lines').new if @output_format == 'json'
+    params['codec'] = LogStash::Plugin.lookup('codec', 'line').new if @output_format == 'plain'
+
     @workers = LogStash::Outputs::Gcs::WorkerPool.new(@max_concurrent_uploads, @upload_synchronous)
     initialize_temp_directory
     initialize_path_factory
@@ -171,25 +179,20 @@ class LogStash::Outputs::GoogleCloudStorage < LogStash::Outputs::Base
     @content_encoding = @gzip_content_encoding ? 'gzip' : 'identity'
   end
 
-  # Method called for each log event. It writes the event to the current output
+  # Method called for incoming log events. It writes the event to the current output
   # file, flushing depending on flush interval configuration.
   public
-  def receive(event)
-    @logger.debug('Received event', :event => event)
-
-    if @output_format == 'json'
-      message = LogStash::Json.dump(event.to_hash)
-    else
-      message = event.to_s
-    end
+  def multi_receive_encoded(event_encoded_pairs)
+    encoded = event_encoded_pairs.map{ |event, encoded| encoded }
+    @logger.debug? && @logger.debug('Received events', :events => encoded)
 
-    @log_rotater.writeln(message)
+    @log_rotater.write(*encoded)
   end
 
   public
   def close
     @logger.debug('Stopping the plugin, uploading the remaining files.')
-    Stud.stop!(@registration_thread) unless @registration_thread.nil?
+    Stud.stop!(@uploader_thread) unless @uploader_thread.nil?
 
     # Force rotate the log. If it contains data it will be submitted
     # to the work pool and will be uploaded before the plugin stops.
@@ -229,9 +232,11 @@ class LogStash::Outputs::GoogleCloudStorage < LogStash::Outputs::Base
 
   # start_uploader periodically sends flush events through the log rotater
   def start_uploader
-    @registration_thread = Thread.new do
+    return if @disable_uploader
+
+    @uploader_thread = Thread.new do
       Stud.interval(@uploader_interval_secs) do
-        @log_rotater.writeln(nil)
+        @log_rotater.write
       end
     end
   end

data/logstash-output-google_cloud_storage.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name            = 'logstash-output-google_cloud_storage'
-  s.version         = '4.0.1'
+  s.version         = '4.1.0'
   s.licenses        = ['Apache-2.0']
   s.summary         = "plugin to upload log events to Google Cloud Storage (GCS)"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
@@ -18,16 +18,18 @@ Gem::Specification.new do |s|
   # Special flag to let us know this is actually a logstash plugin
   s.metadata = { "logstash_plugin" => "true", "logstash_group" => "output" }
 
+  # JARs
+  s.platform = 'java'
+
   # Gem dependencies
   s.add_runtime_dependency "logstash-core-plugin-api", ">= 1.60", "<= 2.99"
 
   s.add_runtime_dependency 'stud'
-  s.add_runtime_dependency 'logstash-codec-plain'
+  s.add_runtime_dependency 'logstash-codec-line'
+  s.add_runtime_dependency 'logstash-codec-json_lines'
   s.add_runtime_dependency 'mime-types', '~> 2' # last version compatible with ruby 2.x
   s.add_runtime_dependency 'concurrent-ruby' # use version bundled with Logstash to avoid platform mismatch on plugin install
-  s.add_development_dependency 'logstash-devutils'
 
-  # JARs
-  s.platform = 'java'
+  s.add_development_dependency 'logstash-devutils'
 end
 

data/spec/outputs/gcs/log_rotate_spec.rb

@@ -27,7 +27,7 @@ describe LogStash::Outputs::Gcs::LogRotate do
     end
   end
 
-  describe '#writeln' do
+  describe '#write' do
     subject { LogStash::Outputs::Gcs::LogRotate.new(path_factory, 10, false, 30) }
 
     it 'does not rotate if size is small and path is the same' do
@@ -35,15 +35,15 @@ describe LogStash::Outputs::Gcs::LogRotate do
       # once for init
       expect(path_factory).to receive(:rotate_path!).once
 
-      subject.writeln('foo')
+      subject.write('foo')
     end
 
     it 'rotates the file if the size is too big' do
       # once for init, once for writeln
       expect(path_factory).to receive(:rotate_path!).twice
 
-      subject.writeln('this line is longer than ten characters' * 1000)
-      subject.writeln('flush')
+      subject.write('this line is longer than ten characters' * 1000)
+      subject.write('flush')
     end
 
     it 'rotates the file if the path changed' do
@@ -51,35 +51,35 @@ describe LogStash::Outputs::Gcs::LogRotate do
       # once for init, once for writeln
       expect(path_factory).to receive(:rotate_path!).twice
 
-      subject.writeln('foo')
+      subject.write('foo')
     end
 
     it 'writes the message' do
       expect(LogStash::Outputs::Gcs::LogFileFactory).to receive(:create).and_return(open_file_1)
-      expect(open_file_1).to receive(:write).with('foo', "\n")
+      expect(open_file_1).to receive(:write).with('foo', 'bar')
 
-      subject.writeln('foo')
+      subject.write('foo', 'bar')
     end
 
-    it 'does not write nil messages' do
+    it 'does not write if there are no parameters' do
       expect(LogStash::Outputs::Gcs::LogFileFactory).to receive(:create).and_return(open_file_1)
       expect(open_file_1).not_to receive(:write)
 
-      subject.writeln(nil)
+      subject.write
     end
 
     it 'does not fsync if delta less than limit' do
       expect(LogStash::Outputs::Gcs::LogFileFactory).to receive(:create).and_return(open_file_1)
       expect(open_file_1).not_to receive(:fsync)
 
-      subject.writeln(nil)
+      subject.write
     end
 
     it 'fsyncs if delta greater than limit' do
       expect(LogStash::Outputs::Gcs::LogFileFactory).to receive(:create).and_return(open_file_2)
       expect(open_file_2).to receive(:fsync)
 
-      subject.writeln(nil)
+      subject.write
     end
   end
 
@@ -106,10 +106,10 @@ describe LogStash::Outputs::Gcs::LogRotate do
 
     it 'opens a new file based on the new path' do
       expect(LogStash::Outputs::Gcs::LogFileFactory).to receive(:create).and_return(open_file_1, open_file_2)
-      expect(open_file_2).to receive(:write).with('foo', "\n")
+      expect(open_file_2).to receive(:write).with('foo', 'bar')
 
       subject.rotate_log!
-      subject.writeln('foo')
+      subject.write('foo', 'bar')
     end
   end
 

data/spec/outputs/gcs/path_factory_spec.rb

@@ -114,13 +114,13 @@ describe LogStash::Outputs::Gcs::PathFactory do
         builder.set_directory 'dir'
         builder.set_prefix 'pre'
         builder.set_include_host false
-        builder.set_date_pattern '%N'
+        builder.set_date_pattern '%s'
         builder.set_include_part true
         builder.set_include_uuid false
         builder.set_is_gzipped false
       end
       expect(pf.current_path).to include('part000')
-
+      sleep(1)
       pf.rotate_path!
       expect(pf.current_path).to include('part000')
     end

data/spec/outputs/google_cloud_storage_spec.rb

@@ -1,6 +1,7 @@
 # encoding: utf-8
 require_relative "../spec_helper"
 require "tempfile"
+require "json"
 
 describe LogStash::Outputs::GoogleCloudStorage do
 
@@ -15,9 +16,86 @@ describe LogStash::Outputs::GoogleCloudStorage do
     allow(javaclient).to receive(:initialize_storage).and_return(:javastorage)
   end
 
-  it "should register without errors" do
+  it 'should register without errors' do
     expect { subject.register }.to_not raise_error
 
     subject.close
   end
+
+  describe '#encode' do
+    it 'should dump the event hash if output_format is json' do
+      encoded = encode_test({
+                                :output_format => 'json',
+                                :event => {'message' => 'contents'}
+                            })
+
+      expect(encoded.end_with?("\n")).to eq(true)
+
+      encoded_hash = JSON.parse(encoded)
+      expect(encoded_hash).to eq({
+                                     'message' => 'contents',
+                                     '@timestamp' => '1970-01-01T00:00:00.000Z',
+                                     'host' => 'localhost',
+                                     '@version' => '1'})
+    end
+
+    it 'should convert to a string if output_format is plain' do
+      encoded = encode_test({
+                      :output_format => 'plain',
+                      :event => {'message' => 'contents'}
+                  })
+
+      expect(encoded).to eq("1970-01-01T00:00:00.000Z localhost contents\n")
+
+    end
+
+    it 'should call the codec if output_format is blank' do
+      encoded = encode_test({
+                      :output_format => nil,
+                      :event => {'message' => 'contents'}
+                  })
+
+      expect(encoded).to eq("1970-01-01T00:00:00.000Z localhost contents\n")
+    end
+
+    it 'should call the codec if no output_format' do
+      encoded = encode_test({
+                                :event => {'message' => 'contents'}
+                            })
+
+      expect(encoded).to eq("1970-01-01T00:00:00.000Z localhost contents\n")
+    end
+  end
+end
+
+def encode_test(params)
+  config = {
+      'bucket' => '',
+      'service_account' => '',
+      'uploader_interval_secs' => 10000,
+      'upload_synchronous' => true,
+  }
+  config['output_format'] = params[:output_format] if params[:output_format]
+
+  rotater = double('rotater')
+  allow(rotater).to receive(:on_rotate)
+  allow(rotater).to receive(:rotate_log!)
+
+  allow(LogStash::Outputs::Gcs::LogRotate).to receive(:new).and_return(rotater)
+
+  gcsout = LogStash::Outputs::GoogleCloudStorage.new(config)
+  gcsout.disable_uploader = true
+  gcsout.register
+
+  event = LogStash::Event.new(params[:event])
+  event.timestamp = LogStash::Timestamp.at(0)
+  event.set('host', 'localhost')
+
+  value = ''
+  allow(rotater).to receive(:write){ |line| value = line }
+
+  gcsout.multi_receive([event])
+  gcsout.close
+
+  value
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-output-google_cloud_storage
 version: !ruby/object:Gem::Version
-  version: 4.0.1
+  version: 4.1.0
 platform: java
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2019-06-24 00:00:00.000000000 Z
+date: 2019-08-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -50,7 +50,21 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-  name: logstash-codec-plain
+  name: logstash-codec-line
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  name: logstash-codec-json_lines
   prerelease: false
   type: :runtime
   version_requirements: !ruby/object:Gem::Requirement