logstash-input-google_cloud_storage 0.9.0-java → 0.10.0-java

data/lib/logstash/inputs/cloud_storage/processed_db.rb

@@ -1,18 +1,5 @@
 # encoding: utf-8
 
-# Copyright 2018 Google LLC
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     https://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
 require 'fileutils'
 require 'digest'
 

data/lib/logstash/inputs/google_cloud_storage.rb

@@ -1,18 +1,5 @@
 # encoding: utf-8
 
-# Copyright 2018 Google LLC
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     https://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
 require 'logstash/inputs/base'
 require 'logstash/namespace'
 require 'stud/interval'
@@ -21,7 +8,7 @@ require 'logstash/inputs/cloud_storage/processed_db'
 require 'logstash/inputs/cloud_storage/blob_filter'
 require 'logstash/inputs/cloud_storage/file_reader'
 
-# GoogleCloudStorage is an input plugin for LogStash that
+# GoogleCloudStorage is an input plugin for Logstash that
 # reads blobs in Cloud Storage buckets.
 class LogStash::Inputs::GoogleCloudStorage < LogStash::Inputs::Base
   config_name 'google_cloud_storage'
@@ -30,7 +17,7 @@ class LogStash::Inputs::GoogleCloudStorage < LogStash::Inputs::Base
 
   # Connection Settings
   config :bucket_id, :validate => :string, :required => true
-  config :json_key_file, :validate => :string, :default => nil
+  config :json_key_file, :validate => :string, :default => ''
   config :interval, :validate => :number, :default => 60
 
   # Inclusion/Exclusion Criteria

data/logstash-input-google_cloud_storage.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name          = 'logstash-input-google_cloud_storage'
-  s.version       = '0.9.0'
+  s.version       = '0.10.0'
   s.licenses      = ['Apache-2.0']
   s.summary       = 'Plugin to import log data from Google Cloud Storage (GCS).'
   s.description   = 'This gem is a Logstash plugin required to be installed on top of the '\

data/spec/inputs/cloud_storage/blob_adapter_spec.rb

@@ -1,19 +1,5 @@
 # encoding: utf-8
 
-# Copyright 2018 Google LLC
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     https://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
 require 'logstash/devutils/rspec/spec_helper'
 require 'logstash/inputs/google_cloud_storage'
 require 'logstash/inputs/cloud_storage/blob_adapter'

data/spec/inputs/cloud_storage/blob_filter_spec.rb

@@ -1,19 +1,5 @@
 # encoding: utf-8
 
-# Copyright 2018 Google LLC
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     https://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
 require 'logstash/devutils/rspec/spec_helper'
 require 'logstash/inputs/google_cloud_storage'
 require 'logstash/inputs/cloud_storage/blob_filter'

data/spec/inputs/cloud_storage/client_spec.rb

@@ -1,19 +1,5 @@
 # encoding: utf-8
 
-# Copyright 2018 Google LLC
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     https://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
 require 'logstash/devutils/rspec/spec_helper'
 require 'logstash/inputs/google_cloud_storage'
 require 'logstash/inputs/cloud_storage/client'
@@ -30,4 +16,4 @@ describe LogStash::Inputs::CloudStorage::Client do
       LogStash::Inputs::CloudStorage::Client.new('my-bucket', key_file, logger)
     end
   end
-end
+end

data/spec/inputs/cloud_storage/file_reader_spec.rb

@@ -1,19 +1,5 @@
 # encoding: utf-8
 
-# Copyright 2018 Google LLC
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     https://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
 require 'logstash/devutils/rspec/spec_helper'
 require 'logstash/inputs/google_cloud_storage'
 require 'logstash/inputs/cloud_storage/file_reader'
@@ -48,7 +34,7 @@ describe LogStash::Inputs::CloudStorage::FileReader do
     let(:path) { ::File.join('spec', 'fixtures', 'helloworld.log') }
 
     it 'reads plain files' do
-      expected = [['hello', 1], ['world', 2]]
+      expected = [["hello\n", 1], ["world\n", 2]]
 
       out = []
       LogStash::Inputs::CloudStorage::FileReader.read_plain_lines(path) do |text, linenum|
@@ -63,7 +49,7 @@ describe LogStash::Inputs::CloudStorage::FileReader do
     let(:path) { ::File.join('spec', 'fixtures', 'helloworld.log.gz') }
 
     it 'reads gzipped files' do
-      expected = [['hello', 1], ['world', 2]]
+      expected = [["hello\n", 1], ["world\n", 2]]
 
       out = []
       LogStash::Inputs::CloudStorage::FileReader.read_gzip_lines(path) do |text, linenum|

data/spec/inputs/cloud_storage/processed_db_spec.rb

@@ -1,19 +1,5 @@
 # encoding: utf-8
 
-# Copyright 2018 Google LLC
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     https://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
 require 'logstash/devutils/rspec/spec_helper'
 require 'logstash/inputs/google_cloud_storage'
 require 'logstash/inputs/cloud_storage/processed_db'

data/spec/inputs/google_cloud_storage_spec.rb

@@ -1,19 +1,5 @@
 # encoding: utf-8
 
-# Copyright 2018 Google LLC
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     https://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
 require 'logstash/devutils/rspec/spec_helper'
 require 'logstash/inputs/google_cloud_storage'
 require 'logstash/inputs/cloud_storage/client'
@@ -83,7 +69,7 @@ describe LogStash::Inputs::GoogleCloudStorage do
       events = subject.event_output_queue
 
       messages = events.map { |e| e.get('message') }
-      expect(messages).to eq(['match.log1', 'match.log2'])
+      expect(messages).to eq(["match.log1\r\n", "match.log2"])
 
       filenames = events.map { |e| e.get('[@metadata][gcs][name]') }
       expect(filenames).to eq(['match.log', 'match.log'])

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: logstash-input-google_cloud_storage
 version: !ruby/object:Gem::Version
-  version: 0.9.0
+  version: 0.10.0
 platform: java
 authors:
 - Elastic
@@ -9,7 +9,7 @@ authors:
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2018-05-25 00:00:00.000000000 Z
+date: 2019-02-28 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -95,11 +95,7 @@ files:
 - Gemfile
 - LICENSE
 - README.md
-- docs/configuration.asciidoc
-- docs/examples.asciidoc
 - docs/index.asciidoc
-- docs/metadata.asciidoc
-- docs/resources.asciidoc
 - lib/logstash-input-google_cloud_storage_jars.rb
 - lib/logstash/inputs/cloud_storage/blob_adapter.rb
 - lib/logstash/inputs/cloud_storage/blob_filter.rb
@@ -117,40 +113,44 @@ files:
 - spec/inputs/cloud_storage/file_reader_spec.rb
 - spec/inputs/cloud_storage/processed_db_spec.rb
 - spec/inputs/google_cloud_storage_spec.rb
-- vendor/jar-dependencies/com/fasterxml/jackson/core/jackson-core/2.1.3/jackson-core-2.1.3.jar
-- vendor/jar-dependencies/com/google/api-client/google-api-client/1.23.0/google-api-client-1.23.0.jar
-- vendor/jar-dependencies/com/google/api/api-common/1.5.0/api-common-1.5.0.jar
-- vendor/jar-dependencies/com/google/api/gax-httpjson/0.40.0/gax-httpjson-0.40.0.jar
-- vendor/jar-dependencies/com/google/api/gax/1.23.0/gax-1.23.0.jar
-- vendor/jar-dependencies/com/google/api/grpc/proto-google-common-protos/1.7.0/proto-google-common-protos-1.7.0.jar
-- vendor/jar-dependencies/com/google/api/grpc/proto-google-iam-v1/0.8.0/proto-google-iam-v1-0.8.0.jar
-- vendor/jar-dependencies/com/google/apis/google-api-services-storage/v1-rev114-1.23.0/google-api-services-storage-v1-rev114-1.23.0.jar
-- vendor/jar-dependencies/com/google/auth/google-auth-library-credentials/0.9.0/google-auth-library-credentials-0.9.0.jar
-- vendor/jar-dependencies/com/google/auth/google-auth-library-oauth2-http/0.9.0/google-auth-library-oauth2-http-0.9.0.jar
-- vendor/jar-dependencies/com/google/cloud/google-cloud-core-http/1.25.0/google-cloud-core-http-1.25.0.jar
-- vendor/jar-dependencies/com/google/cloud/google-cloud-core/1.25.0/google-cloud-core-1.25.0.jar
-- vendor/jar-dependencies/com/google/cloud/google-cloud-storage/1.25.0/google-cloud-storage-1.25.0.jar
-- vendor/jar-dependencies/com/google/code/findbugs/jsr305/3.0.1/jsr305-3.0.1.jar
+- vendor/jar-dependencies/com/fasterxml/jackson/core/jackson-core/2.9.6/jackson-core-2.9.6.jar
+- vendor/jar-dependencies/com/google/api-client/google-api-client/1.27.0/google-api-client-1.27.0.jar
+- vendor/jar-dependencies/com/google/api/api-common/1.7.0/api-common-1.7.0.jar
+- vendor/jar-dependencies/com/google/api/gax-httpjson/0.55.0/gax-httpjson-0.55.0.jar
+- vendor/jar-dependencies/com/google/api/gax/1.38.0/gax-1.38.0.jar
+- vendor/jar-dependencies/com/google/api/grpc/proto-google-common-protos/1.12.0/proto-google-common-protos-1.12.0.jar
+- vendor/jar-dependencies/com/google/api/grpc/proto-google-iam-v1/0.12.0/proto-google-iam-v1-0.12.0.jar
+- vendor/jar-dependencies/com/google/apis/google-api-services-storage/v1-rev20181109-1.27.0/google-api-services-storage-v1-rev20181109-1.27.0.jar
+- vendor/jar-dependencies/com/google/auth/google-auth-library-credentials/0.12.0/google-auth-library-credentials-0.12.0.jar
+- vendor/jar-dependencies/com/google/auth/google-auth-library-oauth2-http/0.12.0/google-auth-library-oauth2-http-0.12.0.jar
+- vendor/jar-dependencies/com/google/cloud/google-cloud-core-http/1.62.0/google-cloud-core-http-1.62.0.jar
+- vendor/jar-dependencies/com/google/cloud/google-cloud-core/1.62.0/google-cloud-core-1.62.0.jar
+- vendor/jar-dependencies/com/google/cloud/google-cloud-storage/1.62.0/google-cloud-storage-1.62.0.jar
+- vendor/jar-dependencies/com/google/code/findbugs/jsr305/3.0.2/jsr305-3.0.2.jar
 - vendor/jar-dependencies/com/google/code/gson/gson/2.7/gson-2.7.jar
 - vendor/jar-dependencies/com/google/errorprone/error_prone_annotations/2.2.0/error_prone_annotations-2.2.0.jar
 - vendor/jar-dependencies/com/google/guava/guava-jdk5/17.0/guava-jdk5-17.0.jar
-- vendor/jar-dependencies/com/google/guava/guava/20.0/guava-20.0.jar
-- vendor/jar-dependencies/com/google/http-client/google-http-client-appengine/1.23.0/google-http-client-appengine-1.23.0.jar
+- vendor/jar-dependencies/com/google/guava/guava/26.0-android/guava-26.0-android.jar
+- vendor/jar-dependencies/com/google/http-client/google-http-client-appengine/1.27.0/google-http-client-appengine-1.27.0.jar
 - vendor/jar-dependencies/com/google/http-client/google-http-client-jackson/1.23.0/google-http-client-jackson-1.23.0.jar
-- vendor/jar-dependencies/com/google/http-client/google-http-client-jackson2/1.23.0/google-http-client-jackson2-1.23.0.jar
-- vendor/jar-dependencies/com/google/http-client/google-http-client/1.23.0/google-http-client-1.23.0.jar
-- vendor/jar-dependencies/com/google/oauth-client/google-oauth-client/1.23.0/google-oauth-client-1.23.0.jar
-- vendor/jar-dependencies/com/google/protobuf/protobuf-java-util/3.5.1/protobuf-java-util-3.5.1.jar
-- vendor/jar-dependencies/com/google/protobuf/protobuf-java/3.5.1/protobuf-java-3.5.1.jar
-- vendor/jar-dependencies/commons-codec/commons-codec/1.3/commons-codec-1.3.jar
-- vendor/jar-dependencies/commons-logging/commons-logging/1.1.1/commons-logging-1.1.1.jar
-- vendor/jar-dependencies/io/grpc/grpc-context/1.9.0/grpc-context-1.9.0.jar
-- vendor/jar-dependencies/io/opencensus/opencensus-api/0.11.1/opencensus-api-0.11.1.jar
-- vendor/jar-dependencies/io/opencensus/opencensus-contrib-http-util/0.11.1/opencensus-contrib-http-util-0.11.1.jar
+- vendor/jar-dependencies/com/google/http-client/google-http-client-jackson2/1.27.0/google-http-client-jackson2-1.27.0.jar
+- vendor/jar-dependencies/com/google/http-client/google-http-client/1.27.0/google-http-client-1.27.0.jar
+- vendor/jar-dependencies/com/google/j2objc/j2objc-annotations/1.1/j2objc-annotations-1.1.jar
+- vendor/jar-dependencies/com/google/oauth-client/google-oauth-client/1.27.0/google-oauth-client-1.27.0.jar
+- vendor/jar-dependencies/com/google/protobuf/protobuf-java-util/3.6.1/protobuf-java-util-3.6.1.jar
+- vendor/jar-dependencies/com/google/protobuf/protobuf-java/3.6.1/protobuf-java-3.6.1.jar
+- vendor/jar-dependencies/commons-codec/commons-codec/1.10/commons-codec-1.10.jar
+- vendor/jar-dependencies/commons-logging/commons-logging/1.2/commons-logging-1.2.jar
+- vendor/jar-dependencies/io/grpc/grpc-context/1.14.0/grpc-context-1.14.0.jar
+- vendor/jar-dependencies/io/opencensus/opencensus-api/0.17.0/opencensus-api-0.17.0.jar
+- vendor/jar-dependencies/io/opencensus/opencensus-contrib-http-util/0.17.0/opencensus-contrib-http-util-0.17.0.jar
+- vendor/jar-dependencies/javax/annotation/javax.annotation-api/1.2/javax.annotation-api-1.2.jar
 - vendor/jar-dependencies/joda-time/joda-time/2.9.2/joda-time-2.9.2.jar
-- vendor/jar-dependencies/org/apache/httpcomponents/httpclient/4.0.1/httpclient-4.0.1.jar
-- vendor/jar-dependencies/org/apache/httpcomponents/httpcore/4.0.1/httpcore-4.0.1.jar
+- vendor/jar-dependencies/org/apache/httpcomponents/httpclient/4.5.5/httpclient-4.5.5.jar
+- vendor/jar-dependencies/org/apache/httpcomponents/httpcore/4.4.9/httpcore-4.4.9.jar
+- vendor/jar-dependencies/org/checkerframework/checker-compat-qual/2.5.2/checker-compat-qual-2.5.2.jar
 - vendor/jar-dependencies/org/codehaus/jackson/jackson-core-asl/1.9.11/jackson-core-asl-1.9.11.jar
+- vendor/jar-dependencies/org/codehaus/mojo/animal-sniffer-annotations/1.14/animal-sniffer-annotations-1.14.jar
 - vendor/jar-dependencies/org/threeten/threetenbp/1.3.3/threetenbp-1.3.3.jar
 homepage: http://www.elastic.co/guide/en/logstash/current/index.html
 licenses:

data/docs/configuration.asciidoc

@@ -1,112 +0,0 @@
-[id="plugins-{type}s-{plugin}-options"]
-==== Google Cloud Storage Input Configuration Options
-
-This plugin supports the following configuration options plus the <<plugins-{type}s-{plugin}-common-options>> described later.
-
-[cols="<,<,<",options="header",]
-|=======================================================================
-|Setting |Input type|Required
-| <<plugins-{type}s-{plugin}-bucket_id>> |<<string,string>>|Yes
-| <<plugins-{type}s-{plugin}-json_key_file>> |<<path,path>>|No
-| <<plugins-{type}s-{plugin}-interval>> |<<number,number>>|No
-| <<plugins-{type}s-{plugin}-file_matches>> |<<string,string>>|No
-| <<plugins-{type}s-{plugin}-file_exclude>> |<<string,string>>|No
-| <<plugins-{type}s-{plugin}-metadata_key>> |<<string,string>>|No
-| <<plugins-{type}s-{plugin}-processed_db_path>> |<<path,path>>|No
-| <<plugins-{type}s-{plugin}-delete>> |<<boolean,boolean>>|No
-| <<plugins-{type}s-{plugin}-unpack_gzip>> |<<boolean,boolean>>|No
-|=======================================================================
-
-Also see <<plugins-{type}s-{plugin}-common-options>> for a list of options supported by all
-input plugins.
-
-
-[id="plugins-{type}s-{plugin}-bucket_id"]
-===== `bucket_id`
-
-  * Value type is <<string,string>>
-  * There is no default value for this setting.
-
-The bucket containing your log files.
-
-[id="plugins-{type}s-{plugin}-json_key_file"]
-===== `json_key_file`
-
-  * Value type is <<path,path>>
-  * There is no default value for this setting.
-
-The path to the key to authenticate your user to the bucket.
-This service user _should_ have the `storage.objects.update` permission so it can
-create metadata on the object preventing it from being scanned multiple times.
-
-[id="plugins-{type}s-{plugin}-interval"]
-===== `interval`
-
-  * Value type is <<number,number>>
-  * Default is: `60`
-
-The number of seconds between looking for new files in your bucket.
-
-[id="plugins-{type}s-{plugin}-file_matches"]
-===== `file_matches`
-
-  * Value type is <<string,string>>
-  * Default is: `.*\.log(\.gz)?`
-
-A regex pattern to filter files. Only files with names matching this will be considered.
-All files match by default.
-
-[id="plugins-{type}s-{plugin}-file_exclude"]
-===== `file_exclude`
-
-  * Value type is <<string,string>>
-  * Default is: `^$`
-
-Any files matching this regex are excluded from processing.
-No files are excluded by default.
-
-[id="plugins-{type}s-{plugin}-metadata_key"]
-===== `metadata_key`
-
-  * Value type is <<string,string>>
-  * Default is: `x-goog-meta-ls-gcs-input`
-
-This key will be set on the objects after they've been processed by the plugin. That way you can
-stop the plugin and not upload files again or prevent them from being uploaded by setting the
-field manually.
-
-NOTE: the key is a flag, if a file was partially processed before Logstash exited some events will be resent.
-
-[id="plugins-{type}s-{plugin}-processed_db_path"]
-===== `processed_db_path`
-
-  * Value type is <<path,path>>
-  * Default is: `LOGSTASH_DATA/plugins/inputs/google_cloud_storage/db`.
-
-If set, the plugin will store the list of processed files locally.
-This allows you to create a service account for the plugin that does not have write permissions.
-However, the data will not be shared across multiple running instances of Logstash.
-
-[id="plugins-{type}s-{plugin}-delete"]
-===== `delete`
-
-  * Value type is <<boolean,boolean>>
-  * Default is: `false`
-
-Should the log file be deleted after its contents have been updated?
-
-[id="plugins-{type}s-{plugin}-unpack_gzip"]
-===== `unpack_gzip`
-
-  * Value type is <<boolean,boolean>>
-  * Default is: `true`
-
-If set to `true`, files ending in `.gz` are decompressed before they're parsed by the codec.
-The file will be skipped if it has the suffix, but can't be opened as a gzip, e.g.
-if it has a bad magic number.
-
-
-[id="plugins-{type}s-{plugin}-common-options"]
-include::{include_path}/{type}.asciidoc[]
-
-:default_codec!: