logstash-input-google_cloud_storage 0.9.0-java

data/spec/inputs/google_cloud_storage_spec.rb

@@ -0,0 +1,127 @@
+# encoding: utf-8
+
+# Copyright 2018 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require 'logstash/devutils/rspec/spec_helper'
+require 'logstash/inputs/google_cloud_storage'
+require 'logstash/inputs/cloud_storage/client'
+require 'logstash/inputs/cloud_storage/blob_filter'
+require 'logstash/inputs/cloud_storage/blob_adapter'
+require 'stud/temporary'
+
+describe LogStash::Inputs::GoogleCloudStorage do
+
+  let(:processed_db_dir) { Stud::Temporary.pathname }
+  let(:download_dir) { Stud::Temporary.pathname }
+
+  let(:config) {
+    {
+        'bucket_id' => 'test-bucket-id',
+        'json_key_file' => ::File.join('spec', 'fixtures', 'credentials.json'),
+        'file_matches' => '.*log',
+        'file_exclude' => 'bak-.*',
+        'metadata_key' => 'test-metadata',
+        'processed_db_path' => processed_db_dir,
+        'temp_directory' => download_dir,
+        'delete' => true,
+        'unpack_gzip' => false
+    }
+  }
+
+  describe '#register' do
+    it 'accepts valid configuration' do
+      subject = LogStash::Inputs::GoogleCloudStorage.new(config)
+      expect { subject.register }.to_not raise_error(RuntimeError)
+    end
+
+    it 'fails with invalid configuration' do
+      bad = config.merge({'file_matches' => ')'})
+      subject = LogStash::Inputs::GoogleCloudStorage.new(bad)
+      expect { subject.register }.to raise_error(RuntimeError)
+    end
+  end
+
+  describe '#list_download_process' do
+    let(:blobs) { mock_blob_list('match.log', 'mismatch.log.xz', 'bak-log.log') }
+    let(:client) do
+      dbl = double('client')
+
+      allow(dbl).to receive(:list_blobs).and_yield(blobs[0]).and_yield(blobs[1]).and_yield(blobs[2])
+
+      dbl
+    end
+
+    subject { LogStash::Inputs::GoogleCloudStorage.new(config) }
+
+    before :each do
+      expect(LogStash::Inputs::CloudStorage::Client).to receive(:new).and_return(client)
+      subject.event_output_queue = []
+      subject.register
+    end
+
+
+    it 'lists files' do
+      expect(subject.processed_db).to receive(:already_processed?).exactly(3).times
+      subject.list_download_process
+    end
+
+    it 'produces events for matching files' do
+      subject.list_download_process
+
+      events = subject.event_output_queue
+
+      messages = events.map { |e| e.get('message') }
+      expect(messages).to eq(['match.log1', 'match.log2'])
+
+      filenames = events.map { |e| e.get('[@metadata][gcs][name]') }
+      expect(filenames).to eq(['match.log', 'match.log'])
+    end
+
+    it 'post-processes matching files' do
+      matching = blobs[0]
+
+      expect(matching).to receive(:set_metadata!)
+      expect(matching).to receive(:delete!)
+
+      subject.list_download_process
+
+      expect(subject.processed_db.already_processed?(matching)).to eq(true)
+    end
+  end
+
+  def mock_blob_list(*names)
+    names.map { |name| mock_blob(name) }
+  end
+
+  def mock_blob(name)
+    generation = rand 2**64
+
+    java_blob = double('Blob',
+                       :getBucket => 'bucket',
+                       :getMetadata => { 'some-tag' => 'true' },
+                       :getName => name,
+                       :getMd5 => 'md5',
+                       :getCrc32c => 'crc',
+                       :getGeneration => generation)
+
+    java_blob.stub_chain(:toBuilder, :setMetadata, :build, :update).and_return(true)
+    java_blob.stub(:downloadTo) do |path|
+      File.open(path.toString, 'w') { |file| file.write("#{name}1\r\n#{name}2") }
+    end
+    java_blob.stub(:delete).and_return(true)
+
+    LogStash::Inputs::CloudStorage::BlobAdapter.new(java_blob)
+  end
+end

metadata

@@ -0,0 +1,191 @@
+--- !ruby/object:Gem::Specification
+name: logstash-input-google_cloud_storage
+version: !ruby/object:Gem::Version
+  version: 0.9.0
+platform: java
+authors:
+- Elastic
+- Joseph Lewis III
+autorequire:
+bindir: bin
+cert_chain: []
+date: 2018-05-25 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  name: logstash-codec-plain
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+  name: logstash-core-plugin-api
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.0.22
+  name: stud
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.0.22
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.0.16
+  name: logstash-devutils
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.0.16
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.3.4
+  name: jar-dependencies
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.3.4
+description: This gem is a Logstash plugin required to be installed on top of the
+  Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This
+  gem is not a stand-alone program
+email: info@elastic.co
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- CHANGELOG.md
+- CONTRIBUTING.md
+- CONTRIBUTORS
+- Gemfile
+- LICENSE
+- README.md
+- docs/configuration.asciidoc
+- docs/examples.asciidoc
+- docs/index.asciidoc
+- docs/metadata.asciidoc
+- docs/resources.asciidoc
+- lib/logstash-input-google_cloud_storage_jars.rb
+- lib/logstash/inputs/cloud_storage/blob_adapter.rb
+- lib/logstash/inputs/cloud_storage/blob_filter.rb
+- lib/logstash/inputs/cloud_storage/client.rb
+- lib/logstash/inputs/cloud_storage/file_reader.rb
+- lib/logstash/inputs/cloud_storage/processed_db.rb
+- lib/logstash/inputs/google_cloud_storage.rb
+- logstash-input-google_cloud_storage.gemspec
+- spec/fixtures/credentials.json
+- spec/fixtures/helloworld.log
+- spec/fixtures/helloworld.log.gz
+- spec/inputs/cloud_storage/blob_adapter_spec.rb
+- spec/inputs/cloud_storage/blob_filter_spec.rb
+- spec/inputs/cloud_storage/client_spec.rb
+- spec/inputs/cloud_storage/file_reader_spec.rb
+- spec/inputs/cloud_storage/processed_db_spec.rb
+- spec/inputs/google_cloud_storage_spec.rb
+- vendor/jar-dependencies/com/fasterxml/jackson/core/jackson-core/2.1.3/jackson-core-2.1.3.jar
+- vendor/jar-dependencies/com/google/api-client/google-api-client/1.23.0/google-api-client-1.23.0.jar
+- vendor/jar-dependencies/com/google/api/api-common/1.5.0/api-common-1.5.0.jar
+- vendor/jar-dependencies/com/google/api/gax-httpjson/0.40.0/gax-httpjson-0.40.0.jar
+- vendor/jar-dependencies/com/google/api/gax/1.23.0/gax-1.23.0.jar
+- vendor/jar-dependencies/com/google/api/grpc/proto-google-common-protos/1.7.0/proto-google-common-protos-1.7.0.jar
+- vendor/jar-dependencies/com/google/api/grpc/proto-google-iam-v1/0.8.0/proto-google-iam-v1-0.8.0.jar
+- vendor/jar-dependencies/com/google/apis/google-api-services-storage/v1-rev114-1.23.0/google-api-services-storage-v1-rev114-1.23.0.jar
+- vendor/jar-dependencies/com/google/auth/google-auth-library-credentials/0.9.0/google-auth-library-credentials-0.9.0.jar
+- vendor/jar-dependencies/com/google/auth/google-auth-library-oauth2-http/0.9.0/google-auth-library-oauth2-http-0.9.0.jar
+- vendor/jar-dependencies/com/google/cloud/google-cloud-core-http/1.25.0/google-cloud-core-http-1.25.0.jar
+- vendor/jar-dependencies/com/google/cloud/google-cloud-core/1.25.0/google-cloud-core-1.25.0.jar
+- vendor/jar-dependencies/com/google/cloud/google-cloud-storage/1.25.0/google-cloud-storage-1.25.0.jar
+- vendor/jar-dependencies/com/google/code/findbugs/jsr305/3.0.1/jsr305-3.0.1.jar
+- vendor/jar-dependencies/com/google/code/gson/gson/2.7/gson-2.7.jar
+- vendor/jar-dependencies/com/google/errorprone/error_prone_annotations/2.2.0/error_prone_annotations-2.2.0.jar
+- vendor/jar-dependencies/com/google/guava/guava-jdk5/17.0/guava-jdk5-17.0.jar
+- vendor/jar-dependencies/com/google/guava/guava/20.0/guava-20.0.jar
+- vendor/jar-dependencies/com/google/http-client/google-http-client-appengine/1.23.0/google-http-client-appengine-1.23.0.jar
+- vendor/jar-dependencies/com/google/http-client/google-http-client-jackson/1.23.0/google-http-client-jackson-1.23.0.jar
+- vendor/jar-dependencies/com/google/http-client/google-http-client-jackson2/1.23.0/google-http-client-jackson2-1.23.0.jar
+- vendor/jar-dependencies/com/google/http-client/google-http-client/1.23.0/google-http-client-1.23.0.jar
+- vendor/jar-dependencies/com/google/oauth-client/google-oauth-client/1.23.0/google-oauth-client-1.23.0.jar
+- vendor/jar-dependencies/com/google/protobuf/protobuf-java-util/3.5.1/protobuf-java-util-3.5.1.jar
+- vendor/jar-dependencies/com/google/protobuf/protobuf-java/3.5.1/protobuf-java-3.5.1.jar
+- vendor/jar-dependencies/commons-codec/commons-codec/1.3/commons-codec-1.3.jar
+- vendor/jar-dependencies/commons-logging/commons-logging/1.1.1/commons-logging-1.1.1.jar
+- vendor/jar-dependencies/io/grpc/grpc-context/1.9.0/grpc-context-1.9.0.jar
+- vendor/jar-dependencies/io/opencensus/opencensus-api/0.11.1/opencensus-api-0.11.1.jar
+- vendor/jar-dependencies/io/opencensus/opencensus-contrib-http-util/0.11.1/opencensus-contrib-http-util-0.11.1.jar
+- vendor/jar-dependencies/joda-time/joda-time/2.9.2/joda-time-2.9.2.jar
+- vendor/jar-dependencies/org/apache/httpcomponents/httpclient/4.0.1/httpclient-4.0.1.jar
+- vendor/jar-dependencies/org/apache/httpcomponents/httpcore/4.0.1/httpcore-4.0.1.jar
+- vendor/jar-dependencies/org/codehaus/jackson/jackson-core-asl/1.9.11/jackson-core-asl-1.9.11.jar
+- vendor/jar-dependencies/org/threeten/threetenbp/1.3.3/threetenbp-1.3.3.jar
+homepage: http://www.elastic.co/guide/en/logstash/current/index.html
+licenses:
+- Apache-2.0
+metadata:
+  logstash_plugin: 'true'
+  logstash_group: input
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+- vendor/jar-dependencies
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project:
+rubygems_version: 2.6.13
+signing_key:
+specification_version: 4
+summary: Plugin to import log data from Google Cloud Storage (GCS).
+test_files:
+- spec/fixtures/credentials.json
+- spec/fixtures/helloworld.log
+- spec/fixtures/helloworld.log.gz
+- spec/inputs/cloud_storage/blob_adapter_spec.rb
+- spec/inputs/cloud_storage/blob_filter_spec.rb
+- spec/inputs/cloud_storage/client_spec.rb
+- spec/inputs/cloud_storage/file_reader_spec.rb
+- spec/inputs/cloud_storage/processed_db_spec.rb
+- spec/inputs/google_cloud_storage_spec.rb