logstash-output-google_cloud_storage 4.0.1-java → 4.2.0-java

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 4aa7c73beda983444233b7943c788a49c2a9a7ac56218e78dce4bbe1932fb39a
-  data.tar.gz: 7f88b0005ff2fada77f84d55365693e3a9eb582178923bdf240fe66db3f15ef0
+  metadata.gz: b38c4e1b40e1211060b087f636ef4fc0300bba164886e074cc67a03ad66f29a9
+  data.tar.gz: 458ff02f9e010e64c7a76b10c915bc1a8f12c18aca60fe6deeae3d558596ea13
 SHA512:
-  metadata.gz: dddd13b7892f549aa748efcfd4a8bc16f108bb840c5179006ce12b57f027d55cafc5ed18ad36c03d7c2e2a370460ed1ebec26252323096cda3407eda5158dd94
-  data.tar.gz: 2195cf539ad95c824a0c1f4f93527c76c6ff9c0ae5c6b9c98d44670004372ef49bd5fdfc843461e3d6879f11818a5bb5ad5103efde785193484ea4b73dcc07a9
+  metadata.gz: 5b86d137f987254387c242f79b523b83c67bd5221fcdcdc83c3b8cdb4d6b5a3bfcd5aec63c86f2430e58e84b7b997791a555ec0d14f4aec004bd5ea5dfe3433b
+  data.tar.gz: 512f0215310dfcf5a67471029e77ebb76c24104ea36109cb704ded739a35c737b5fa7fe10c36a2426dda776893697fc0679cc96179e57dbd2d20eff23e3efb07

data/CHANGELOG.md

@@ -1,3 +1,9 @@
+## 4.2.0
+  - Updated Google Cloud Storage client library, needs Logstash `>= 8.3.0` to run [#48](https://github.com/logstash-plugins/logstash-output-google_cloud_storage/pull/48)
+
+## 4.1.0
+  - Added ability to use Logstash codecs [#42](https://github.com/logstash-plugins/logstash-output-google_cloud_storage/pull/42)
+
 ## 4.0.1
   - Fixes [#38](https://github.com/logstash-plugins/logstash-output-google_cloud_storage/issues/38) - Plugin doesn't start on logstash 7.1.1 - TypeError
 

data/CONTRIBUTORS

@@ -3,6 +3,7 @@ reports, or in general have helped logstash along its way.
 
 Contributors:
 * Aaron Mildenstein (untergeek)
+* Colin Surprenant (colinsurprenant)
 * Ethan Estrada (eestrada)
 * Google LLC.
 * Jordan Sissel (jordansissel)

data/LICENSE

@@ -1,13 +1,202 @@
-Copyright (c) 2012-2018 Elasticsearch <http://www.elastic.co>
 
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
 
-    http://www.apache.org/licenses/LICENSE-2.0
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
 
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright 2020 Elastic and contributors
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

data/README.md

@@ -1,6 +1,6 @@
 # Logstash Plugin
 
-[![Travis Build Status](https://travis-ci.org/logstash-plugins/logstash-output-google_cloud_storage.svg)](https://travis-ci.org/logstash-plugins/logstash-output-google_cloud_storage)
+[![Travis Build Status](https://travis-ci.com/logstash-plugins/logstash-output-google_cloud_storage.svg)](https://travis-ci.com/logstash-plugins/logstash-output-google_cloud_storage)
 
 This is a plugin for [Logstash](https://github.com/elastic/logstash).
 

data/docs/index.asciidoc

@@ -1,6 +1,6 @@
 :plugin: google_cloud_storage
 :type: output
-:default_codec: plain
+:default_codec: line
 
 ///////////////////////////////////////////
 START - GENERATED VARIABLES, DO NOT EDIT!
@@ -101,7 +101,7 @@ This plugin supports the following configuration options plus the <<plugins-{typ
 | <<plugins-{type}s-{plugin}-log_file_prefix>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-max_concurrent_uploads>> |<<number,number>>|No
 | <<plugins-{type}s-{plugin}-max_file_size_kbytes>> |<<number,number>>|No
-| <<plugins-{type}s-{plugin}-output_format>> |<<string,string>>, one of `["json", "plain"]`|No
+| <<plugins-{type}s-{plugin}-output_format>> |<<string,string>>, one of `["json", "plain", nil]`|__Deprecated__
 | <<plugins-{type}s-{plugin}-service_account>> |<<string,string>>|__Deprecated__
 | <<plugins-{type}s-{plugin}-temp_directory>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-uploader_interval_secs>> |<<number,number>>|No
@@ -264,11 +264,18 @@ Sets max file size in kbytes. 0 disable max file check.
 [id="plugins-{type}s-{plugin}-output_format"]
 ===== `output_format` 
 
-  * Value can be any of: `json`, `plain`
-  * Default value is `"plain"`
+  * Value can be any of: `json`, `plain`, or no value
+  * Default value is no value
+
+**Deprecated**, this feature will be removed in the next major release. Use codecs instead.
+
+  * If you are using the `json` value today, switch to the `json_lines` codec.
+  * If you are using the `plain` value today, switch to the `line` codec.
 
 The event format you want to store in files. Defaults to plain text.
 
+Note: if you want to use a codec you MUST not set this value.
+
 [id="plugins-{type}s-{plugin}-service_account"]
 ===== `service_account` 
 

data/lib/logstash/outputs/gcs/log_rotate.rb

@@ -19,15 +19,15 @@ module LogStash
           rotate_log!
         end
 
-        # writeln writes a message and carriage-return character to the open
-        # log file, rotating and syncing it if necessary.
+        # write writes zero or more messages to the open log file
+        # rotating and syncing it if necessary.
         #
         # nil messages do not get written, but may cause the log to rotate
-        def writeln(message=nil)
+        def write(*messages)
           @lock.with_write_lock do
             rotate_log! if should_rotate?
 
-            @temp_file.write(message, "\n") unless message.nil?
+            @temp_file.write(*messages) unless messages.empty?
 
             @temp_file.fsync if @temp_file.time_since_sync >= @flush_interval_secs
           end

data/lib/logstash/outputs/google_cloud_storage.rb

@@ -58,7 +58,6 @@ require "zlib"
 #      temp_directory => "/tmp/logstash-gcs"                     (optional)
 #      log_file_prefix => "logstash_gcs"                         (optional)
 #      max_file_size_kbytes => 1024                              (optional)
-#      output_format => "plain"                                  (optional)
 #      date_pattern => "%Y-%m-%dT%H:00"                          (optional)
 #      flush_interval_secs => 2                                  (optional)
 #      gzip => false                                             (optional)
@@ -79,6 +78,7 @@ class LogStash::Outputs::GoogleCloudStorage < LogStash::Outputs::Base
   config_name "google_cloud_storage"
 
   concurrency :single
+  default :codec, "line"
 
   # GCS bucket name, without "gs://" or any other prefix.
   config :bucket, :validate => :string, :required => true
@@ -104,7 +104,7 @@ class LogStash::Outputs::GoogleCloudStorage < LogStash::Outputs::Base
   config :max_file_size_kbytes, :validate => :number, :default => 10000
 
   # The event format you want to store in files. Defaults to plain text.
-  config :output_format, :validate => [ "json", "plain" ], :default => "plain"
+  config :output_format, :validate => [ "json", "plain", nil ], :default => nil, :deprecated => 'Use codec instead.'
 
   # Time pattern for log file, defaults to hourly files.
   # Must Time.strftime patterns: www.ruby-doc.org/core-2.0/Time.html#method-i-strftime
@@ -154,10 +154,17 @@ class LogStash::Outputs::GoogleCloudStorage < LogStash::Outputs::Base
 
   config :max_concurrent_uploads, :validate  => :number, :default => 5
 
-  public
+  attr_accessor :disable_uploader, :active
+
   def register
     @logger.debug('Registering Google Cloud Storage plugin')
 
+    # NOTE: this is a hacky solution to get around the fact that we used to
+    # do our own pseudo-codec processing. This should be removed in the
+    # next major release.
+    params['codec'] = LogStash::Plugin.lookup('codec', 'json_lines').new if @output_format == 'json'
+    params['codec'] = LogStash::Plugin.lookup('codec', 'line').new if @output_format == 'plain'
+
     @workers = LogStash::Outputs::Gcs::WorkerPool.new(@max_concurrent_uploads, @upload_synchronous)
     initialize_temp_directory
     initialize_path_factory
@@ -171,25 +178,18 @@ class LogStash::Outputs::GoogleCloudStorage < LogStash::Outputs::Base
     @content_encoding = @gzip_content_encoding ? 'gzip' : 'identity'
   end
 
-  # Method called for each log event. It writes the event to the current output
+  # Method called for incoming log events. It writes the event to the current output
   # file, flushing depending on flush interval configuration.
-  public
-  def receive(event)
-    @logger.debug('Received event', :event => event)
-
-    if @output_format == 'json'
-      message = LogStash::Json.dump(event.to_hash)
-    else
-      message = event.to_s
-    end
+  def multi_receive_encoded(event_encoded_pairs)
+    encoded = event_encoded_pairs.map{ |event, encoded| encoded }
+    @logger.debug? && @logger.debug('Received events', :events => encoded)
 
-    @log_rotater.writeln(message)
+    @log_rotater.write(*encoded)
   end
 
-  public
   def close
     @logger.debug('Stopping the plugin, uploading the remaining files.')
-    Stud.stop!(@registration_thread) unless @registration_thread.nil?
+    Stud.stop!(@uploader_thread) unless @uploader_thread.nil?
 
     # Force rotate the log. If it contains data it will be submitted
     # to the work pool and will be uploaded before the plugin stops.
@@ -229,9 +229,11 @@ class LogStash::Outputs::GoogleCloudStorage < LogStash::Outputs::Base
 
   # start_uploader periodically sends flush events through the log rotater
   def start_uploader
-    @registration_thread = Thread.new do
+    return if @disable_uploader
+
+    @uploader_thread = Thread.new do
       Stud.interval(@uploader_interval_secs) do
-        @log_rotater.writeln(nil)
+        @log_rotater.write
       end
     end
   end
@@ -272,5 +274,4 @@ class LogStash::Outputs::GoogleCloudStorage < LogStash::Outputs::Base
       end
     end
   end
-  attr_accessor :active
 end

data/lib/logstash-output-google_cloud_storage_jars.rb

@@ -1,35 +1,39 @@
 # AUTOGENERATED BY THE GRADLE SCRIPT. DO NOT EDIT.
 
 require 'jar_dependencies'
-require_jar('com.google.cloud', 'google-cloud-storage', '1.39.0')
-require_jar('com.google.api', 'api-common', '1.7.0')
-require_jar('com.google.api-client', 'google-api-client', '1.24.1')
-require_jar('com.google.api', 'gax', '1.30.0')
-require_jar('com.google.api', 'gax-httpjson', '0.47.0')
-require_jar('com.google.api.grpc', 'proto-google-common-protos', '1.12.0')
-require_jar('com.google.api.grpc', 'proto-google-iam-v1', '0.12.0')
-require_jar('com.google.apis', 'google-api-services-storage', 'v1-rev135-1.24.1')
-require_jar('com.google.auth', 'google-auth-library-credentials', '0.10.0')
-require_jar('com.google.auth', 'google-auth-library-oauth2-http', '0.10.0')
-require_jar('com.google.cloud', 'google-cloud-core', '1.39.0')
-require_jar('com.google.cloud', 'google-cloud-core-http', '1.39.0')
+require_jar('com.google.cloud', 'google-cloud-storage', '1.118.1')
+require_jar('com.google.guava', 'guava', '30.1.1-jre')
+require_jar('com.google.guava', 'failureaccess', '1.0.1')
+require_jar('com.google.guava', 'listenablefuture', '9999.0-empty-to-avoid-conflict-with-guava')
 require_jar('com.google.code.findbugs', 'jsr305', '3.0.2')
-require_jar('com.google.code.gson', 'gson', '2.7')
-require_jar('com.google.guava', 'guava', '20.0')
-require_jar('com.google.http-client', 'google-http-client', '1.24.1')
-require_jar('com.google.http-client', 'google-http-client-appengine', '1.24.1')
-require_jar('com.google.http-client', 'google-http-client-jackson', '1.24.1')
-require_jar('com.google.http-client', 'google-http-client-jackson2', '1.24.1')
-require_jar('com.google.oauth-client', 'google-oauth-client', '1.24.1')
-require_jar('com.google.protobuf', 'protobuf-java', '3.6.0')
-require_jar('com.google.protobuf', 'protobuf-java-util', '3.6.0')
-require_jar('commons-codec', 'commons-codec', '1.9')
-require_jar('commons-logging', 'commons-logging', '1.2')
-require_jar('io.grpc', 'grpc-context', '1.12.0')
-require_jar('io.opencensus', 'opencensus-api', '0.15.0')
-require_jar('io.opencensus', 'opencensus-contrib-http-util', '0.15.0')
-require_jar('joda-time', 'joda-time', '2.9.2')
-require_jar('org.apache.httpcomponents', 'httpclient', '4.5.3')
-require_jar('org.apache.httpcomponents', 'httpcore', '4.4.6')
-require_jar('org.codehaus.jackson', 'jackson-core-asl', '1.9.11')
-require_jar('org.threeten', 'threetenbp', '1.3.3')
+require_jar('org.checkerframework', 'checker-qual', '3.8.0')
+require_jar('com.google.errorprone', 'error_prone_annotations', '2.8.1')
+require_jar('com.google.j2objc', 'j2objc-annotations', '1.3')
+require_jar('com.google.http-client', 'google-http-client', '1.39.2')
+require_jar('io.opencensus', 'opencensus-contrib-http-util', '0.28.0')
+require_jar('com.google.http-client', 'google-http-client-jackson2', '1.39.2')
+require_jar('com.fasterxml.jackson.core', 'jackson-core', '2.13.3')
+require_jar('com.google.api-client', 'google-api-client', '1.32.1')
+require_jar('com.google.oauth-client', 'google-oauth-client', '1.31.5')
+require_jar('com.google.http-client', 'google-http-client-gson', '1.39.2')
+require_jar('com.google.http-client', 'google-http-client-apache-v2', '1.39.2')
+require_jar('com.google.apis', 'google-api-services-storage', 'v1-rev20210127-1.32.1')
+require_jar('com.google.code.gson', 'gson', '2.8.7')
+require_jar('com.google.cloud', 'google-cloud-core', '2.0.2')
+require_jar('com.google.auto.value', 'auto-value-annotations', '1.8.2')
+require_jar('com.google.api.grpc', 'proto-google-common-protos', '2.3.2')
+require_jar('com.google.cloud', 'google-cloud-core-http', '2.0.2')
+require_jar('com.google.http-client', 'google-http-client-appengine', '1.39.2')
+require_jar('com.google.api', 'gax-httpjson', '0.85.0')
+require_jar('com.google.api', 'gax', '2.0.0')
+require_jar('com.google.auth', 'google-auth-library-credentials', '1.0.0')
+require_jar('com.google.auth', 'google-auth-library-oauth2-http', '1.0.0')
+require_jar('com.google.api', 'api-common', '2.0.0')
+require_jar('javax.annotation', 'javax.annotation-api', '1.3.2')
+require_jar('io.opencensus', 'opencensus-api', '0.28.0')
+require_jar('io.grpc', 'grpc-context', '1.39.0')
+require_jar('com.google.api.grpc', 'proto-google-iam-v1', '1.0.14')
+require_jar('com.google.protobuf', 'protobuf-java', '3.17.3')
+require_jar('com.google.protobuf', 'protobuf-java-util', '3.17.3')
+require_jar('org.threeten', 'threetenbp', '1.5.1')
+require_jar('org.logstash.inputs', 'plugin', '4.2.0')

data/logstash-output-google_cloud_storage.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name            = 'logstash-output-google_cloud_storage'
-  s.version         = '4.0.1'
+  s.version         = ::File.read('version').split("\n").first
   s.licenses        = ['Apache-2.0']
   s.summary         = "plugin to upload log events to Google Cloud Storage (GCS)"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
@@ -10,7 +10,7 @@ Gem::Specification.new do |s|
   s.require_paths = ["lib", "vendor/jar-dependencies"]
 
   # Files
-  s.files = Dir["lib/**/*","spec/**/*","*.gemspec","*.md","CONTRIBUTORS","Gemfile","LICENSE","NOTICE.TXT", "vendor/jar-dependencies/**/*.jar", "vendor/jar-dependencies/**/*.rb", "VERSION", "docs/**/*"]
+  s.files = Dir["lib/**/*","spec/**/*","*.gemspec","*.md","CONTRIBUTORS","Gemfile","LICENSE","NOTICE.TXT", "vendor/jar-dependencies/**/*.jar", "vendor/jar-dependencies/**/*.rb", "version", "docs/**/*"]
 
   # Tests
   s.test_files = s.files.grep(%r{^(test|spec|features)/})
@@ -18,16 +18,19 @@ Gem::Specification.new do |s|
   # Special flag to let us know this is actually a logstash plugin
   s.metadata = { "logstash_plugin" => "true", "logstash_group" => "output" }
 
+  # JARs
+  s.platform = 'java'
+
   # Gem dependencies
+  s.add_runtime_dependency 'logstash-core', '>= 8.3.0'
   s.add_runtime_dependency "logstash-core-plugin-api", ">= 1.60", "<= 2.99"
 
   s.add_runtime_dependency 'stud'
-  s.add_runtime_dependency 'logstash-codec-plain'
+  s.add_runtime_dependency 'logstash-codec-line'
+  s.add_runtime_dependency 'logstash-codec-json_lines'
   s.add_runtime_dependency 'mime-types', '~> 2' # last version compatible with ruby 2.x
   s.add_runtime_dependency 'concurrent-ruby' # use version bundled with Logstash to avoid platform mismatch on plugin install
-  s.add_development_dependency 'logstash-devutils'
 
-  # JARs
-  s.platform = 'java'
+  s.add_development_dependency 'logstash-devutils'
 end
 

data/spec/outputs/gcs/log_rotate_spec.rb

@@ -27,7 +27,7 @@ describe LogStash::Outputs::Gcs::LogRotate do
     end
   end
 
-  describe '#writeln' do
+  describe '#write' do
     subject { LogStash::Outputs::Gcs::LogRotate.new(path_factory, 10, false, 30) }
 
     it 'does not rotate if size is small and path is the same' do
@@ -35,15 +35,15 @@ describe LogStash::Outputs::Gcs::LogRotate do
       # once for init
       expect(path_factory).to receive(:rotate_path!).once
 
-      subject.writeln('foo')
+      subject.write('foo')
     end
 
     it 'rotates the file if the size is too big' do
       # once for init, once for writeln
       expect(path_factory).to receive(:rotate_path!).twice
 
-      subject.writeln('this line is longer than ten characters' * 1000)
-      subject.writeln('flush')
+      subject.write('this line is longer than ten characters' * 1000)
+      subject.write('flush')
     end
 
     it 'rotates the file if the path changed' do
@@ -51,35 +51,35 @@ describe LogStash::Outputs::Gcs::LogRotate do
       # once for init, once for writeln
       expect(path_factory).to receive(:rotate_path!).twice
 
-      subject.writeln('foo')
+      subject.write('foo')
     end
 
     it 'writes the message' do
       expect(LogStash::Outputs::Gcs::LogFileFactory).to receive(:create).and_return(open_file_1)
-      expect(open_file_1).to receive(:write).with('foo', "\n")
+      expect(open_file_1).to receive(:write).with('foo', 'bar')
 
-      subject.writeln('foo')
+      subject.write('foo', 'bar')
     end
 
-    it 'does not write nil messages' do
+    it 'does not write if there are no parameters' do
       expect(LogStash::Outputs::Gcs::LogFileFactory).to receive(:create).and_return(open_file_1)
       expect(open_file_1).not_to receive(:write)
 
-      subject.writeln(nil)
+      subject.write
     end
 
     it 'does not fsync if delta less than limit' do
       expect(LogStash::Outputs::Gcs::LogFileFactory).to receive(:create).and_return(open_file_1)
       expect(open_file_1).not_to receive(:fsync)
 
-      subject.writeln(nil)
+      subject.write
     end
 
     it 'fsyncs if delta greater than limit' do
       expect(LogStash::Outputs::Gcs::LogFileFactory).to receive(:create).and_return(open_file_2)
       expect(open_file_2).to receive(:fsync)
 
-      subject.writeln(nil)
+      subject.write
     end
   end
 
@@ -106,10 +106,10 @@ describe LogStash::Outputs::Gcs::LogRotate do
 
     it 'opens a new file based on the new path' do
       expect(LogStash::Outputs::Gcs::LogFileFactory).to receive(:create).and_return(open_file_1, open_file_2)
-      expect(open_file_2).to receive(:write).with('foo', "\n")
+      expect(open_file_2).to receive(:write).with('foo', 'bar')
 
       subject.rotate_log!
-      subject.writeln('foo')
+      subject.write('foo', 'bar')
     end
   end
 

data/spec/outputs/gcs/path_factory_spec.rb

@@ -114,13 +114,13 @@ describe LogStash::Outputs::Gcs::PathFactory do
         builder.set_directory 'dir'
         builder.set_prefix 'pre'
         builder.set_include_host false
-        builder.set_date_pattern '%N'
+        builder.set_date_pattern '%s'
         builder.set_include_part true
         builder.set_include_uuid false
         builder.set_is_gzipped false
       end
       expect(pf.current_path).to include('part000')
-
+      sleep(1)
       pf.rotate_path!
       expect(pf.current_path).to include('part000')
     end

data/spec/outputs/google_cloud_storage_spec.rb

@@ -1,6 +1,7 @@
 # encoding: utf-8
 require_relative "../spec_helper"
 require "tempfile"
+require "json"
 
 describe LogStash::Outputs::GoogleCloudStorage do
 
@@ -15,9 +16,85 @@ describe LogStash::Outputs::GoogleCloudStorage do
     allow(javaclient).to receive(:initialize_storage).and_return(:javastorage)
   end
 
-  it "should register without errors" do
+  it 'should register without errors' do
     expect { subject.register }.to_not raise_error
 
     subject.close
   end
+
+  describe '#encode' do
+    it 'should dump the event hash if output_format is json' do
+      encoded = encode_test({
+                                :output_format => 'json',
+                                :event => {'message' => 'contents'}
+                            })
+
+      expect(encoded.end_with?("\n")).to eq(true)
+
+      encoded_hash = JSON.parse(encoded)
+      expect(encoded_hash['message']).to eq('contents')
+      expect(encoded_hash['@timestamp']).to match(/1970-01-01T00:00:00(\.000)?Z/)
+      expect(encoded_hash['host']).to eq('localhost')
+      expect(encoded_hash['@version']).to eq('1')
+     end
+
+    it 'should convert to a string if output_format is plain' do
+      encoded = encode_test({
+                      :output_format => 'plain',
+                      :event => {'message' => 'contents'}
+                  })
+
+      expect(encoded).to match(/1970-01-01T00:00:00(\.000)?Z localhost contents\n/)
+
+    end
+
+    it 'should call the codec if output_format is blank' do
+      encoded = encode_test({
+                      :output_format => nil,
+                      :event => {'message' => 'contents'}
+                  })
+
+      expect(encoded).to match(/1970-01-01T00:00:00(\.000)?Z localhost contents\n/)
+    end
+
+    it 'should call the codec if no output_format' do
+      encoded = encode_test({
+                                :event => {'message' => 'contents'}
+                            })
+
+      expect(encoded).to match(/1970-01-01T00:00:00(\.000)?Z localhost contents\n/)
+    end
+  end
+end
+
+def encode_test(params)
+  config = {
+      'bucket' => '',
+      'service_account' => '',
+      'uploader_interval_secs' => 10000,
+      'upload_synchronous' => true,
+  }
+  config['output_format'] = params[:output_format] if params[:output_format]
+
+  rotater = double('rotater')
+  allow(rotater).to receive(:on_rotate)
+  allow(rotater).to receive(:rotate_log!)
+
+  allow(LogStash::Outputs::Gcs::LogRotate).to receive(:new).and_return(rotater)
+
+  gcsout = LogStash::Outputs::GoogleCloudStorage.new(config)
+  gcsout.disable_uploader = true
+  gcsout.register
+
+  event = LogStash::Event.new(params[:event])
+  event.timestamp = LogStash::Timestamp.at(0)
+  event.set('host', 'localhost')
+
+  value = ''
+  allow(rotater).to receive(:write){ |line| value = line }
+
+  gcsout.multi_receive([event])
+  gcsout.close
+
+  value
 end

data/version

@@ -0,0 +1 @@
+4.2.0

metadata

@@ -1,15 +1,29 @@
 --- !ruby/object:Gem::Specification
 name: logstash-output-google_cloud_storage
 version: !ruby/object:Gem::Version
-  version: 4.0.1
+  version: 4.2.0
 platform: java
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2019-06-24 00:00:00.000000000 Z
+date: 2022-09-23 00:00:00.000000000 Z
 dependencies:
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 8.3.0
+  name: logstash-core
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 8.3.0
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
@@ -50,7 +64,21 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-  name: logstash-codec-plain
+  name: logstash-codec-line
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  name: logstash-codec-json_lines
   prerelease: false
   type: :runtime
   version_requirements: !ruby/object:Gem::Requirement
@@ -131,38 +159,43 @@ files:
 - spec/outputs/gcs/worker_pool_spec.rb
 - spec/outputs/google_cloud_storage_spec.rb
 - spec/spec_helper.rb
-- vendor/jar-dependencies/com/google/api-client/google-api-client/1.24.1/google-api-client-1.24.1.jar
-- vendor/jar-dependencies/com/google/api/api-common/1.7.0/api-common-1.7.0.jar
-- vendor/jar-dependencies/com/google/api/gax-httpjson/0.47.0/gax-httpjson-0.47.0.jar
-- vendor/jar-dependencies/com/google/api/gax/1.30.0/gax-1.30.0.jar
-- vendor/jar-dependencies/com/google/api/grpc/proto-google-common-protos/1.12.0/proto-google-common-protos-1.12.0.jar
-- vendor/jar-dependencies/com/google/api/grpc/proto-google-iam-v1/0.12.0/proto-google-iam-v1-0.12.0.jar
-- vendor/jar-dependencies/com/google/apis/google-api-services-storage/v1-rev135-1.24.1/google-api-services-storage-v1-rev135-1.24.1.jar
-- vendor/jar-dependencies/com/google/auth/google-auth-library-credentials/0.10.0/google-auth-library-credentials-0.10.0.jar
-- vendor/jar-dependencies/com/google/auth/google-auth-library-oauth2-http/0.10.0/google-auth-library-oauth2-http-0.10.0.jar
-- vendor/jar-dependencies/com/google/cloud/google-cloud-core-http/1.39.0/google-cloud-core-http-1.39.0.jar
-- vendor/jar-dependencies/com/google/cloud/google-cloud-core/1.39.0/google-cloud-core-1.39.0.jar
-- vendor/jar-dependencies/com/google/cloud/google-cloud-storage/1.39.0/google-cloud-storage-1.39.0.jar
+- vendor/jar-dependencies/com/fasterxml/jackson/core/jackson-core/2.13.3/jackson-core-2.13.3.jar
+- vendor/jar-dependencies/com/google/api-client/google-api-client/1.32.1/google-api-client-1.32.1.jar
+- vendor/jar-dependencies/com/google/api/api-common/2.0.0/api-common-2.0.0.jar
+- vendor/jar-dependencies/com/google/api/gax-httpjson/0.85.0/gax-httpjson-0.85.0.jar
+- vendor/jar-dependencies/com/google/api/gax/2.0.0/gax-2.0.0.jar
+- vendor/jar-dependencies/com/google/api/grpc/proto-google-common-protos/2.3.2/proto-google-common-protos-2.3.2.jar
+- vendor/jar-dependencies/com/google/api/grpc/proto-google-iam-v1/1.0.14/proto-google-iam-v1-1.0.14.jar
+- vendor/jar-dependencies/com/google/apis/google-api-services-storage/v1-rev20210127-1.32.1/google-api-services-storage-v1-rev20210127-1.32.1.jar
+- vendor/jar-dependencies/com/google/auth/google-auth-library-credentials/1.0.0/google-auth-library-credentials-1.0.0.jar
+- vendor/jar-dependencies/com/google/auth/google-auth-library-oauth2-http/1.0.0/google-auth-library-oauth2-http-1.0.0.jar
+- vendor/jar-dependencies/com/google/auto/value/auto-value-annotations/1.8.2/auto-value-annotations-1.8.2.jar
+- vendor/jar-dependencies/com/google/cloud/google-cloud-core-http/2.0.2/google-cloud-core-http-2.0.2.jar
+- vendor/jar-dependencies/com/google/cloud/google-cloud-core/2.0.2/google-cloud-core-2.0.2.jar
+- vendor/jar-dependencies/com/google/cloud/google-cloud-storage/1.118.1/google-cloud-storage-1.118.1.jar
 - vendor/jar-dependencies/com/google/code/findbugs/jsr305/3.0.2/jsr305-3.0.2.jar
-- vendor/jar-dependencies/com/google/code/gson/gson/2.7/gson-2.7.jar
-- vendor/jar-dependencies/com/google/guava/guava/20.0/guava-20.0.jar
-- vendor/jar-dependencies/com/google/http-client/google-http-client-appengine/1.24.1/google-http-client-appengine-1.24.1.jar
-- vendor/jar-dependencies/com/google/http-client/google-http-client-jackson/1.24.1/google-http-client-jackson-1.24.1.jar
-- vendor/jar-dependencies/com/google/http-client/google-http-client-jackson2/1.24.1/google-http-client-jackson2-1.24.1.jar
-- vendor/jar-dependencies/com/google/http-client/google-http-client/1.24.1/google-http-client-1.24.1.jar
-- vendor/jar-dependencies/com/google/oauth-client/google-oauth-client/1.24.1/google-oauth-client-1.24.1.jar
-- vendor/jar-dependencies/com/google/protobuf/protobuf-java-util/3.6.0/protobuf-java-util-3.6.0.jar
-- vendor/jar-dependencies/com/google/protobuf/protobuf-java/3.6.0/protobuf-java-3.6.0.jar
-- vendor/jar-dependencies/commons-codec/commons-codec/1.9/commons-codec-1.9.jar
-- vendor/jar-dependencies/commons-logging/commons-logging/1.2/commons-logging-1.2.jar
-- vendor/jar-dependencies/io/grpc/grpc-context/1.12.0/grpc-context-1.12.0.jar
-- vendor/jar-dependencies/io/opencensus/opencensus-api/0.15.0/opencensus-api-0.15.0.jar
-- vendor/jar-dependencies/io/opencensus/opencensus-contrib-http-util/0.15.0/opencensus-contrib-http-util-0.15.0.jar
-- vendor/jar-dependencies/joda-time/joda-time/2.9.2/joda-time-2.9.2.jar
-- vendor/jar-dependencies/org/apache/httpcomponents/httpclient/4.5.3/httpclient-4.5.3.jar
-- vendor/jar-dependencies/org/apache/httpcomponents/httpcore/4.4.6/httpcore-4.4.6.jar
-- vendor/jar-dependencies/org/codehaus/jackson/jackson-core-asl/1.9.11/jackson-core-asl-1.9.11.jar
-- vendor/jar-dependencies/org/threeten/threetenbp/1.3.3/threetenbp-1.3.3.jar
+- vendor/jar-dependencies/com/google/code/gson/gson/2.8.7/gson-2.8.7.jar
+- vendor/jar-dependencies/com/google/errorprone/error_prone_annotations/2.8.1/error_prone_annotations-2.8.1.jar
+- vendor/jar-dependencies/com/google/guava/failureaccess/1.0.1/failureaccess-1.0.1.jar
+- vendor/jar-dependencies/com/google/guava/guava/30.1.1-jre/guava-30.1.1-jre.jar
+- vendor/jar-dependencies/com/google/guava/listenablefuture/9999.0-empty-to-avoid-conflict-with-guava/listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar
+- vendor/jar-dependencies/com/google/http-client/google-http-client-apache-v2/1.39.2/google-http-client-apache-v2-1.39.2.jar
+- vendor/jar-dependencies/com/google/http-client/google-http-client-appengine/1.39.2/google-http-client-appengine-1.39.2.jar
+- vendor/jar-dependencies/com/google/http-client/google-http-client-gson/1.39.2/google-http-client-gson-1.39.2.jar
+- vendor/jar-dependencies/com/google/http-client/google-http-client-jackson2/1.39.2/google-http-client-jackson2-1.39.2.jar
+- vendor/jar-dependencies/com/google/http-client/google-http-client/1.39.2/google-http-client-1.39.2.jar
+- vendor/jar-dependencies/com/google/j2objc/j2objc-annotations/1.3/j2objc-annotations-1.3.jar
+- vendor/jar-dependencies/com/google/oauth-client/google-oauth-client/1.31.5/google-oauth-client-1.31.5.jar
+- vendor/jar-dependencies/com/google/protobuf/protobuf-java-util/3.17.3/protobuf-java-util-3.17.3.jar
+- vendor/jar-dependencies/com/google/protobuf/protobuf-java/3.17.3/protobuf-java-3.17.3.jar
+- vendor/jar-dependencies/io/grpc/grpc-context/1.39.0/grpc-context-1.39.0.jar
+- vendor/jar-dependencies/io/opencensus/opencensus-api/0.28.0/opencensus-api-0.28.0.jar
+- vendor/jar-dependencies/io/opencensus/opencensus-contrib-http-util/0.28.0/opencensus-contrib-http-util-0.28.0.jar
+- vendor/jar-dependencies/javax/annotation/javax.annotation-api/1.3.2/javax.annotation-api-1.3.2.jar
+- vendor/jar-dependencies/org/checkerframework/checker-qual/3.8.0/checker-qual-3.8.0.jar
+- vendor/jar-dependencies/org/logstash/inputs/plugin/4.2.0/plugin-4.2.0.jar
+- vendor/jar-dependencies/org/threeten/threetenbp/1.5.1/threetenbp-1.5.1.jar
+- version
 homepage: http://www.elastic.co/guide/en/logstash/current/index.html
 licenses:
 - Apache-2.0
@@ -185,8 +218,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project:
-rubygems_version: 2.6.13
+rubygems_version: 3.1.6
 signing_key:
 specification_version: 4
 summary: plugin to upload log events to Google Cloud Storage (GCS)