logstash-input-google_cloud_storage 0.12.0-java → 0.14.0-java

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 32368da3607b542514c07135118a2ac728807663d043026f1a16b83ab7ab8572
-  data.tar.gz: eb51485cd0d3de22909be15ef8c5ed27ee79633c7fa0b47a2be4133ed60bd2d8
+  metadata.gz: d5deb7203755cea341c28b9ae42f8e62605c0c4ac3880ae1ecb58fd4f92cda63
+  data.tar.gz: dba78aed232fa56d67341ad5e4e3dd6a7361d42c2df2ae040f4795cc63dbcd90
 SHA512:
-  metadata.gz: 0add9d92e3794ca0aa8872ac5dc7a340ece9c49a01b480537ef40768dff1db69ace6ce95d7929de606ddf61a92a8913950cebcb3e9e988333c8d8b7d2242b18b
-  data.tar.gz: feb07220ba68ec36951ab0dd68284ac5f04b718c355477c6c23315ab41cea628c5f6a2b8aec14bbd357b16898b6443304091396377c4f3dad4658ab36e8ed8c9
+  metadata.gz: 93c2a0b32fea57e63490d67bd3ff6995d58c8f1dcceec472435d073751d5aed8bfd0202a5e42cf872562c3d9d798cc88d0cf6ebdfba3cd6652c53c18654a1786
+  data.tar.gz: f9edbf2e175a54ac412bff1fb6b516381875c8a72188bfbbe2c8039905c64a70238ca7a89432d090edfe09ec2fad86a0261c37da62d66d20f0c354c17cd732b8

data/CHANGELOG.md

@@ -1,3 +1,9 @@
+## 0.14.0
+ - Feature, update so that unauthenticated client can access public buckets [#24](https://github.com/logstash-plugins/logstash-input-google_cloud_storage/pull/24)
+
+## 0.13.0
+ - Updated Google Cloud Storage client library, from `1.118.1` to `2.12.0` [#21](https://github.com/logstash-plugins/logstash-output-google_cloud_storage/pull/21)
+
 ## 0.12.0
  - Updated Google Cloud Storage client library, needs Logstash `>= 8.3.0` to run [#20](https://github.com/logstash-plugins/logstash-input-google_cloud_storage/pull/20)
 

data/README.md

@@ -26,19 +26,29 @@ Need help? Try #logstash on freenode IRC or the https://discuss.elastic.co/c/log
 
 ## Developing
 
-### 1. Plugin Developement and Testing
+### 1. Plugin Development and Testing
 
 #### Code
+
 - To get started, you'll need JRuby with the Bundler gem installed.
+- You'll also need a Logstash installation to build the plugin against.
 
 - Create a new plugin or clone and existing from the GitHub [logstash-plugins](https://github.com/logstash-plugins) organization. We also provide [example plugins](https://github.com/logstash-plugins?query=example).
+- `export LOGSTASH_SOURCE=1` and point `LOGSTASH_PATH` to a local Logstash
+  e.g. `export LOGSTASH_PATH=/opt/local/logstash-8.7.0`
 
-- Install dependencies
+- Install Ruby dependencies
 ```sh
 bundle install
-bundle exec rake vendor
 ```
 
+- Install Java dependencies - regenerates the *lib/logstash-input-google_cloud_storage_jars.rb*
+  script used to load the .jar dependencies when the plugin starts.
+```sh
+./gradlew vendor
+```
+  NOTE: This step is necessary whenever **build.gradle** is updated.
+
 #### Test
 
 - Update your dependencies
@@ -47,7 +57,7 @@ bundle exec rake vendor
 bundle install
 ```
 
-- Run tests
+- Run Ruby tests
 
 ```sh
 bundle exec rspec

data/docs/index.asciidoc

@@ -56,6 +56,8 @@ To manually install the "mimemagic" gem into Logstash use:
 bin/ruby -S gem install mimemagic -v '>= 0.3.7'
 ----------------------------------
 
+The mimemagic gem also requires the `shared-mime-info` package to be present, it can be installed using `apt-get install shared-mime-info` on Debian/Ubuntu or `yum install shared-mime-info` on Red Hat/RockyOS distributions.
+
 Then install the plugin as usual with:
 [source,bash]
 ----------------------------------
@@ -217,8 +219,9 @@ The bucket containing your log files.
   * There is no default value for this setting.
 
 The path to the key to authenticate your user to the bucket.
-This service user _should_ have the `storage.objects.update` permission so it can
-create metadata on the object preventing it from being scanned multiple times.
+This service user _should_ have the `storage.objects.update` permission so it can create metadata on the object preventing it from being scanned multiple times.
+
+If no key is provided the plugin will try to use the https://cloud.google.com/java/docs/reference/google-auth-library/latest/com.google.auth.oauth2.GoogleCredentials#com_google_auth_oauth2_GoogleCredentials_getApplicationDefault__[default application credentials], and if they don't exist, it falls back to unauthenticated mode. 
 
 [id="plugins-{type}s-{plugin}-interval"]
 ===== `interval`

data/lib/logstash/inputs/cloud_storage/client.rb

@@ -27,24 +27,41 @@ module LogStash
         end
 
         private
-
+        
+        
         def initialize_storage(json_key_path)
-          com.google.cloud.storage.StorageOptions.newBuilder()
-             .setCredentials(credentials(json_key_path))
-             .setHeaderProvider(http_headers)
-             .setRetrySettings(retry_settings)
-             .build()
-             .getService()
+          builder = setup_builder_from_credentials(json_key_path)
+          builder.setHeaderProvider(http_headers)
+              .setRetrySettings(retry_settings)
+              .build()
+              .getService()
         end
-
+        
         java_import 'com.google.auth.oauth2.GoogleCredentials'
-        def credentials(json_key_path)
-          return GoogleCredentials.getApplicationDefault() if json_key_path.empty?
+        def setup_builder_from_credentials(json_key_path)
+          # initialize the StorageOptions builder
+          builder = com.google.cloud.storage.StorageOptions.newBuilder()
 
-          key_file = java.io.FileInputStream.new(json_key_path)
-          GoogleCredentials.fromStream(key_file)
+          # initialize it normally if a json_key_path is provided
+          if !json_key_path.empty?
+            key_file = java.io.FileInputStream.new(json_key_path)
+            builder.setCredentials(GoogleCredentials.fromStream(key_file))
+          else
+            # if a json_key_path is not provided, try using the getApplicationDefault, normally 
+            # using GOOGLE_APPLICATION_CREDENTIALS env variable 
+            begin
+              builder.setCredentials(GoogleCredentials.getApplicationDefault())
+            rescue Java::JavaIo::IOException => e
+              # an IOException is generated if no default credentials exist, trying unauthenticated
+              builder = com.google.cloud.storage.StorageOptions.getUnauthenticatedInstance()
+              .toBuilder()
+            end
+          end
+          return builder
         end
 
+        
+
         java_import 'com.google.api.gax.rpc.FixedHeaderProvider'
         def http_headers
           gem_name = 'logstash-input-google_cloud_storage'

data/lib/logstash-input-google_cloud_storage_jars.rb

@@ -1,39 +1,38 @@
 # AUTOGENERATED BY THE GRADLE SCRIPT. DO NOT EDIT.
 
 require 'jar_dependencies'
-require_jar('com.google.cloud', 'google-cloud-storage', '1.118.1')
-require_jar('com.google.guava', 'guava', '30.1.1-jre')
+require_jar('com.google.cloud', 'google-cloud-storage', '2.12.0')
+require_jar('com.google.guava', 'guava', '31.1-jre')
 require_jar('com.google.guava', 'failureaccess', '1.0.1')
 require_jar('com.google.guava', 'listenablefuture', '9999.0-empty-to-avoid-conflict-with-guava')
-require_jar('com.google.code.findbugs', 'jsr305', '3.0.2')
-require_jar('org.checkerframework', 'checker-qual', '3.8.0')
-require_jar('com.google.errorprone', 'error_prone_annotations', '2.8.1')
+require_jar('org.checkerframework', 'checker-qual', '3.25.0')
 require_jar('com.google.j2objc', 'j2objc-annotations', '1.3')
-require_jar('com.google.http-client', 'google-http-client', '1.39.2')
-require_jar('io.opencensus', 'opencensus-contrib-http-util', '0.28.0')
-require_jar('com.google.http-client', 'google-http-client-jackson2', '1.39.2')
-require_jar('com.fasterxml.jackson.core', 'jackson-core', '2.12.4')
-require_jar('com.google.api-client', 'google-api-client', '1.32.1')
-require_jar('com.google.oauth-client', 'google-oauth-client', '1.31.5')
-require_jar('com.google.http-client', 'google-http-client-gson', '1.39.2')
-require_jar('com.google.http-client', 'google-http-client-apache-v2', '1.39.2')
-require_jar('com.google.apis', 'google-api-services-storage', 'v1-rev20210127-1.32.1')
-require_jar('com.google.code.gson', 'gson', '2.8.7')
-require_jar('com.google.cloud', 'google-cloud-core', '2.0.2')
-require_jar('com.google.auto.value', 'auto-value-annotations', '1.8.2')
-require_jar('com.google.api.grpc', 'proto-google-common-protos', '2.3.2')
-require_jar('com.google.cloud', 'google-cloud-core-http', '2.0.2')
-require_jar('com.google.http-client', 'google-http-client-appengine', '1.39.2')
-require_jar('com.google.api', 'gax-httpjson', '0.85.0')
-require_jar('com.google.api', 'gax', '2.0.0')
-require_jar('com.google.auth', 'google-auth-library-credentials', '1.0.0')
-require_jar('com.google.auth', 'google-auth-library-oauth2-http', '1.0.0')
-require_jar('com.google.api', 'api-common', '2.0.0')
+require_jar('com.google.http-client', 'google-http-client', '1.42.2')
+require_jar('io.opencensus', 'opencensus-contrib-http-util', '0.31.1')
+require_jar('com.google.http-client', 'google-http-client-jackson2', '1.42.2')
+require_jar('com.google.api-client', 'google-api-client', '2.0.0')
+require_jar('com.google.oauth-client', 'google-oauth-client', '1.34.1')
+require_jar('com.google.http-client', 'google-http-client-gson', '1.42.2')
+require_jar('com.google.http-client', 'google-http-client-apache-v2', '1.42.2')
+require_jar('com.google.apis', 'google-api-services-storage', 'v1-rev20220705-2.0.0')
+require_jar('com.google.code.gson', 'gson', '2.9.1')
+require_jar('com.google.cloud', 'google-cloud-core', '2.8.12')
+require_jar('com.google.auto.value', 'auto-value-annotations', '1.9')
+require_jar('com.google.api.grpc', 'proto-google-common-protos', '2.9.2')
+require_jar('com.google.cloud', 'google-cloud-core-http', '2.8.12')
+require_jar('com.google.http-client', 'google-http-client-appengine', '1.42.2')
+require_jar('com.google.api', 'gax-httpjson', '0.104.1')
+require_jar('com.google.api', 'gax', '2.19.1')
+require_jar('com.google.auth', 'google-auth-library-credentials', '1.11.0')
+require_jar('com.google.auth', 'google-auth-library-oauth2-http', '1.11.0')
+require_jar('com.google.api', 'api-common', '2.2.1')
 require_jar('javax.annotation', 'javax.annotation-api', '1.3.2')
-require_jar('io.opencensus', 'opencensus-api', '0.28.0')
-require_jar('io.grpc', 'grpc-context', '1.39.0')
-require_jar('com.google.api.grpc', 'proto-google-iam-v1', '1.0.14')
-require_jar('com.google.protobuf', 'protobuf-java', '3.17.3')
-require_jar('com.google.protobuf', 'protobuf-java-util', '3.17.3')
-require_jar('org.threeten', 'threetenbp', '1.5.1')
-require_jar('org.logstash.inputs', 'plugin', '0.12.0')
+require_jar('io.opencensus', 'opencensus-api', '0.31.1')
+require_jar('io.grpc', 'grpc-context', '1.49.0')
+require_jar('com.google.api.grpc', 'proto-google-iam-v1', '1.5.2')
+require_jar('com.google.protobuf', 'protobuf-java', '3.21.6')
+require_jar('com.google.protobuf', 'protobuf-java-util', '3.21.6')
+require_jar('org.threeten', 'threetenbp', '1.6.1')
+require_jar('com.fasterxml.jackson.core', 'jackson-core', '2.13.4')
+require_jar('com.google.code.findbugs', 'jsr305', '3.0.2')
+require_jar('org.logstash.inputs', 'plugin', '0.14.0')

data/version

@@ -1 +1 @@
-0.12.0
+0.14.0

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: logstash-input-google_cloud_storage
 version: !ruby/object:Gem::Version
-  version: 0.12.0
+  version: 0.14.0
 platform: java
 authors:
 - Elastic
@@ -9,7 +9,7 @@ authors:
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2022-10-04 00:00:00.000000000 Z
+date: 2023-05-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -149,42 +149,41 @@ files:
 - spec/inputs/cloud_storage/file_reader_spec.rb
 - spec/inputs/cloud_storage/processed_db_spec.rb
 - spec/inputs/google_cloud_storage_spec.rb
-- vendor/jar-dependencies/com/fasterxml/jackson/core/jackson-core/2.12.4/jackson-core-2.12.4.jar
-- vendor/jar-dependencies/com/google/api-client/google-api-client/1.32.1/google-api-client-1.32.1.jar
-- vendor/jar-dependencies/com/google/api/api-common/2.0.0/api-common-2.0.0.jar
-- vendor/jar-dependencies/com/google/api/gax-httpjson/0.85.0/gax-httpjson-0.85.0.jar
-- vendor/jar-dependencies/com/google/api/gax/2.0.0/gax-2.0.0.jar
-- vendor/jar-dependencies/com/google/api/grpc/proto-google-common-protos/2.3.2/proto-google-common-protos-2.3.2.jar
-- vendor/jar-dependencies/com/google/api/grpc/proto-google-iam-v1/1.0.14/proto-google-iam-v1-1.0.14.jar
-- vendor/jar-dependencies/com/google/apis/google-api-services-storage/v1-rev20210127-1.32.1/google-api-services-storage-v1-rev20210127-1.32.1.jar
-- vendor/jar-dependencies/com/google/auth/google-auth-library-credentials/1.0.0/google-auth-library-credentials-1.0.0.jar
-- vendor/jar-dependencies/com/google/auth/google-auth-library-oauth2-http/1.0.0/google-auth-library-oauth2-http-1.0.0.jar
-- vendor/jar-dependencies/com/google/auto/value/auto-value-annotations/1.8.2/auto-value-annotations-1.8.2.jar
-- vendor/jar-dependencies/com/google/cloud/google-cloud-core-http/2.0.2/google-cloud-core-http-2.0.2.jar
-- vendor/jar-dependencies/com/google/cloud/google-cloud-core/2.0.2/google-cloud-core-2.0.2.jar
-- vendor/jar-dependencies/com/google/cloud/google-cloud-storage/1.118.1/google-cloud-storage-1.118.1.jar
+- vendor/jar-dependencies/com/fasterxml/jackson/core/jackson-core/2.13.4/jackson-core-2.13.4.jar
+- vendor/jar-dependencies/com/google/api-client/google-api-client/2.0.0/google-api-client-2.0.0.jar
+- vendor/jar-dependencies/com/google/api/api-common/2.2.1/api-common-2.2.1.jar
+- vendor/jar-dependencies/com/google/api/gax-httpjson/0.104.1/gax-httpjson-0.104.1.jar
+- vendor/jar-dependencies/com/google/api/gax/2.19.1/gax-2.19.1.jar
+- vendor/jar-dependencies/com/google/api/grpc/proto-google-common-protos/2.9.2/proto-google-common-protos-2.9.2.jar
+- vendor/jar-dependencies/com/google/api/grpc/proto-google-iam-v1/1.5.2/proto-google-iam-v1-1.5.2.jar
+- vendor/jar-dependencies/com/google/apis/google-api-services-storage/v1-rev20220705-2.0.0/google-api-services-storage-v1-rev20220705-2.0.0.jar
+- vendor/jar-dependencies/com/google/auth/google-auth-library-credentials/1.11.0/google-auth-library-credentials-1.11.0.jar
+- vendor/jar-dependencies/com/google/auth/google-auth-library-oauth2-http/1.11.0/google-auth-library-oauth2-http-1.11.0.jar
+- vendor/jar-dependencies/com/google/auto/value/auto-value-annotations/1.9/auto-value-annotations-1.9.jar
+- vendor/jar-dependencies/com/google/cloud/google-cloud-core-http/2.8.12/google-cloud-core-http-2.8.12.jar
+- vendor/jar-dependencies/com/google/cloud/google-cloud-core/2.8.12/google-cloud-core-2.8.12.jar
+- vendor/jar-dependencies/com/google/cloud/google-cloud-storage/2.12.0/google-cloud-storage-2.12.0.jar
 - vendor/jar-dependencies/com/google/code/findbugs/jsr305/3.0.2/jsr305-3.0.2.jar
-- vendor/jar-dependencies/com/google/code/gson/gson/2.8.7/gson-2.8.7.jar
-- vendor/jar-dependencies/com/google/errorprone/error_prone_annotations/2.8.1/error_prone_annotations-2.8.1.jar
+- vendor/jar-dependencies/com/google/code/gson/gson/2.9.1/gson-2.9.1.jar
 - vendor/jar-dependencies/com/google/guava/failureaccess/1.0.1/failureaccess-1.0.1.jar
-- vendor/jar-dependencies/com/google/guava/guava/30.1.1-jre/guava-30.1.1-jre.jar
+- vendor/jar-dependencies/com/google/guava/guava/31.1-jre/guava-31.1-jre.jar
 - vendor/jar-dependencies/com/google/guava/listenablefuture/9999.0-empty-to-avoid-conflict-with-guava/listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar
-- vendor/jar-dependencies/com/google/http-client/google-http-client-apache-v2/1.39.2/google-http-client-apache-v2-1.39.2.jar
-- vendor/jar-dependencies/com/google/http-client/google-http-client-appengine/1.39.2/google-http-client-appengine-1.39.2.jar
-- vendor/jar-dependencies/com/google/http-client/google-http-client-gson/1.39.2/google-http-client-gson-1.39.2.jar
-- vendor/jar-dependencies/com/google/http-client/google-http-client-jackson2/1.39.2/google-http-client-jackson2-1.39.2.jar
-- vendor/jar-dependencies/com/google/http-client/google-http-client/1.39.2/google-http-client-1.39.2.jar
+- vendor/jar-dependencies/com/google/http-client/google-http-client-apache-v2/1.42.2/google-http-client-apache-v2-1.42.2.jar
+- vendor/jar-dependencies/com/google/http-client/google-http-client-appengine/1.42.2/google-http-client-appengine-1.42.2.jar
+- vendor/jar-dependencies/com/google/http-client/google-http-client-gson/1.42.2/google-http-client-gson-1.42.2.jar
+- vendor/jar-dependencies/com/google/http-client/google-http-client-jackson2/1.42.2/google-http-client-jackson2-1.42.2.jar
+- vendor/jar-dependencies/com/google/http-client/google-http-client/1.42.2/google-http-client-1.42.2.jar
 - vendor/jar-dependencies/com/google/j2objc/j2objc-annotations/1.3/j2objc-annotations-1.3.jar
-- vendor/jar-dependencies/com/google/oauth-client/google-oauth-client/1.31.5/google-oauth-client-1.31.5.jar
-- vendor/jar-dependencies/com/google/protobuf/protobuf-java-util/3.17.3/protobuf-java-util-3.17.3.jar
-- vendor/jar-dependencies/com/google/protobuf/protobuf-java/3.17.3/protobuf-java-3.17.3.jar
-- vendor/jar-dependencies/io/grpc/grpc-context/1.39.0/grpc-context-1.39.0.jar
-- vendor/jar-dependencies/io/opencensus/opencensus-api/0.28.0/opencensus-api-0.28.0.jar
-- vendor/jar-dependencies/io/opencensus/opencensus-contrib-http-util/0.28.0/opencensus-contrib-http-util-0.28.0.jar
+- vendor/jar-dependencies/com/google/oauth-client/google-oauth-client/1.34.1/google-oauth-client-1.34.1.jar
+- vendor/jar-dependencies/com/google/protobuf/protobuf-java-util/3.21.6/protobuf-java-util-3.21.6.jar
+- vendor/jar-dependencies/com/google/protobuf/protobuf-java/3.21.6/protobuf-java-3.21.6.jar
+- vendor/jar-dependencies/io/grpc/grpc-context/1.49.0/grpc-context-1.49.0.jar
+- vendor/jar-dependencies/io/opencensus/opencensus-api/0.31.1/opencensus-api-0.31.1.jar
+- vendor/jar-dependencies/io/opencensus/opencensus-contrib-http-util/0.31.1/opencensus-contrib-http-util-0.31.1.jar
 - vendor/jar-dependencies/javax/annotation/javax.annotation-api/1.3.2/javax.annotation-api-1.3.2.jar
-- vendor/jar-dependencies/org/checkerframework/checker-qual/3.8.0/checker-qual-3.8.0.jar
-- vendor/jar-dependencies/org/logstash/inputs/plugin/0.12.0/plugin-0.12.0.jar
-- vendor/jar-dependencies/org/threeten/threetenbp/1.5.1/threetenbp-1.5.1.jar
+- vendor/jar-dependencies/org/checkerframework/checker-qual/3.25.0/checker-qual-3.25.0.jar
+- vendor/jar-dependencies/org/logstash/inputs/plugin/0.14.0/plugin-0.14.0.jar
+- vendor/jar-dependencies/org/threeten/threetenbp/1.6.1/threetenbp-1.6.1.jar
 - version
 homepage: http://www.elastic.co/guide/en/logstash/current/index.html
 licenses: