logstash-input-google_cloud_storage 0.9.0-java

data/logstash-input-google_cloud_storage.gemspec

@@ -0,0 +1,49 @@
+Gem::Specification.new do |s|
+  s.name          = 'logstash-input-google_cloud_storage'
+  s.version       = '0.9.0'
+  s.licenses      = ['Apache-2.0']
+  s.summary       = 'Plugin to import log data from Google Cloud Storage (GCS).'
+  s.description   = 'This gem is a Logstash plugin required to be installed on top of the '\
+                    'Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. '\
+                    'This gem is not a stand-alone program'
+  s.homepage      = 'http://www.elastic.co/guide/en/logstash/current/index.html'
+  s.authors       = ['Elastic', 'Joseph Lewis III']
+  s.email         = 'info@elastic.co'
+  s.require_paths = ['lib', 'vendor/jar-dependencies']
+
+  # Files
+  s.files = Dir[
+      # Code
+      'lib/**/*',
+      'spec/**/*',
+      'vendor/**/*',
+      'vendor/jar-dependencies/**/*.jar',
+      'vendor/jar-dependencies/**/*.rb',
+      # Library
+      'Gemfile',
+      '*.gemspec',
+      # Documentation
+      '*.md',
+      'CONTRIBUTORS',
+      'LICENSE',
+      'NOTICE.TXT',
+      'VERSION',
+      'docs/**/*']
+
+  # Tests
+  s.test_files = s.files.grep(%r{^(test|spec|features)/})
+
+  # Special flag to let us know this is actually a logstash plugin
+  s.metadata = { 'logstash_plugin' => 'true', 'logstash_group' => 'input' }
+
+  # Gem dependencies
+  s.add_runtime_dependency 'logstash-codec-plain'
+  s.add_runtime_dependency 'logstash-core-plugin-api', '~> 2.0'
+  s.add_runtime_dependency 'stud', '>= 0.0.22'
+
+  s.add_development_dependency 'logstash-devutils', '>= 0.0.16'
+
+  # Java
+  s.add_development_dependency 'jar-dependencies', '~> 0.3.4'
+  s.platform = 'java'
+end

data/spec/fixtures/credentials.json

@@ -0,0 +1,8 @@
+{
+  "//": "Dummy account from https://github.com/GoogleCloudPlatform/google-cloud-java/google-cloud-clients/google-cloud-core/src/test/java/com/google/cloud/ServiceOptionsTest.java",
+  "private_key_id": "somekeyid",
+  "private_key": "-----BEGIN PRIVATE KEY-----\nMIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQC+K2hSuFpAdrJI\nnCgcDz2M7t7bjdlsadsasad+fvRSW6TjNQZ3p5LLQY1kSZRqBqylRkzteMOyHgaR\n0Pmxh3ILCND5men43j3h4eDbrhQBuxfEMalkG92sL+PNQSETY2tnvXryOvmBRwa/\nQP/9dJfIkIDJ9Fw9N4Bhhhp6mCcRpdQjV38H7JsyJ7lih/oNjECgYAt\nknddadwkwewcVxHFhcZJO+XWf6ofLUXpRwiTZakGMn8EE1uVa2LgczOjwWHGi99MFjxSer5m9\n1tCa3/KEGKiS/YL71JvjwX3mb+cewlkcmweBKZHM2JPTk0ZednFSpVZMtycjkbLa\ndYOS8V85AgMBewECggEBAKksaldajfDZDV6nGqbFjMiizAKJolr/M3OQw16K6o3/\n0S31xIe3sSlgW0+UbYlF4U8KifhManD1apVSC3csafaspP4RZUHFhtBywLO9pR5c\nr6S5aLp+gPWFyIp1pfXbWGvc5VY/v9x7ya1VEa6rXvLsKupSeWAW4tMj3eo/64ge\nsdaceaLYw52KeBYiT6+vpsnYrEkAHO1fF/LavbLLOFJmFTMxmsNaG0tuiJHgjshB\n82DpMCbXG9YcCgI/DbzuIjsdj2JC1cascSP//3PmefWysucBQe7Jryb6NQtASmnv\nCdDw/0jmZTEjpe4S1lxfHplAhHFtdgYTvyYtaLZiVVkCgYEA8eVpof2rceecw/I6\n5ng1q3Hl2usdWV/4mZMvR0fOemacLLfocX6IYxT1zA1FFJlbXSRsJMf/Qq39mOR2\nSpW+hr4jCoHeRVYLgsbggtrevGmILAlNoqCMpGZ6vDmJpq6ECV9olliDvpPgWOP+\nmYPDreFBGxWvQrADNbRt2dmGsrsCgYEAyUHqB2wvJHFqdmeBsaacewzV8x9WgmeX\ngUIi9REwXlGDW0Mz50dxpxcKCAYn65+7TCnY5O/jmL0VRxU1J2mSWyWTo1C+17L0\n3fUqjxL1pkefwecxwecvC+gFFYdJ4CQ/MHHXU81Lwl1iWdFCd2UoGddYaOF+KNeM\nHC7cmqra+JsCgYEAlUNywzq8nUg7282E+uICfCB0LfwejuymR93CtsFgb7cRd6ak\nECR8FGfCpH8ruWJINllbQfcHVCX47ndLZwqv3oVFKh6pAS/vVI4dpOepP8++7y1u\ncoOvtreXCX6XqfrWDtKIvv0vjlHBhhhp6mCcRpdQjV38H7JsyJ7lih/oNjECgYAt\nkndj5uNl5SiuVxHFhcZJO+XWf6ofLUregtevZakGMn8EE1uVa2AY7eafmoU/nZPT\n00YB0TBATdCbn/nBSuKDESkhSg9s2GEKQZG5hBmL5uCMfo09z3SfxZIhJdlerreP\nJ7gSidI12N+EZxYd4xIJh/HFDgp7RRO87f+WJkofMQKBgGTnClK1VMaCRbJZPriw\nEfeFCoOX75MxKwXs6xgrw4W//AYGGUjDt83lD6AZP6tws7gJ2IwY/qP7+lyhjEqN\nHtfPZRGFkGZsdaksdlaksd323423d+15/UvrlRSFPNj1tWQmNKkXyRDW4IG1Oa2p\nrALStNBx5Y9t0/LQnFI4w3aG\n-----END PRIVATE KEY-----\n",
+  "client_email": "someclientid@developer.gserviceaccount.com",
+  "client_id": "someclientid.apps.googleusercontent.com",
+  "type": "service_account"
+}

data/spec/fixtures/helloworld.log

@@ -0,0 +1,2 @@
+hello
+world

data/spec/inputs/cloud_storage/blob_adapter_spec.rb

@@ -0,0 +1,95 @@
+# encoding: utf-8
+
+# Copyright 2018 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require 'logstash/devutils/rspec/spec_helper'
+require 'logstash/inputs/google_cloud_storage'
+require 'logstash/inputs/cloud_storage/blob_adapter'
+
+describe LogStash::Inputs::CloudStorage::BlobAdapter do
+  let(:blob) do
+    double('Blob',
+      :getBucket => 'bucket',
+      :getMetadata => {},
+      :getName => 'name',
+      :getMd5 => 'md5',
+      :getCrc32c => 'crc',
+      :getGeneration => 123,
+      :downloadTo => true)
+  end
+  subject { LogStash::Inputs::CloudStorage::BlobAdapter.new(blob) }
+
+  describe '#attributes' do
+    it 'includes the bucket ID as bucket' do
+      expect(subject.attributes).to have_key('bucket')
+    end
+
+    it 'includes metadata as metadata' do
+      expect(subject.attributes).to have_key('metadata')
+    end
+
+    it 'includes the blob name as name' do
+      expect(subject.attributes).to have_key('name')
+    end
+
+    it 'includes the md5 as md5' do
+      expect(subject.attributes).to have_key('md5')
+    end
+
+    it 'includes the crc32c as crc32c' do
+      expect(subject.attributes).to have_key('crc32c')
+    end
+
+    it 'includes the generation as generation' do
+      expect(subject.attributes).to have_key('generation')
+    end
+  end
+
+  describe '#line_attributes' do
+    it 'sets line_id to the correct format' do
+      attrs = subject.line_attributes(9876)
+      expect(attrs['line_id']).to eq('gs://bucket/name:9876@123')
+    end
+
+    it 'includes line' do
+      attrs = subject.line_attributes(9876)
+      expect(attrs['line']).to eq(9876)
+    end
+  end
+
+  describe '#with_downloaded' do
+    before(:each) do
+      allow(FileUtils).to receive(:remove_entry_secure)
+    end
+
+    it 'uses temp_directory as a base' do
+      expect { |b| subject.with_downloaded('foo', &b) }.to yield_with_args(/foo.*/)
+    end
+
+    it 'calls downloadTo on the blob' do
+      expect(blob).to receive(:downloadTo)
+      subject.with_downloaded('foo') {}
+    end
+
+    it 'yields the downloaded file' do
+      expect { |b| subject.with_downloaded('foo', &b) }.to yield_with_args(String)
+    end
+
+    it 'securely deletes the file once done' do
+      expect(FileUtils).to receive(:remove_entry_secure)
+      subject.with_downloaded('foo') {}
+    end
+  end
+end

data/spec/inputs/cloud_storage/blob_filter_spec.rb

@@ -0,0 +1,92 @@
+# encoding: utf-8
+
+# Copyright 2018 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require 'logstash/devutils/rspec/spec_helper'
+require 'logstash/inputs/google_cloud_storage'
+require 'logstash/inputs/cloud_storage/blob_filter'
+require 'logstash/inputs/cloud_storage/blob_adapter'
+
+describe LogStash::Inputs::CloudStorage::BlobFilter do
+  describe '#initialize' do
+    it 'should fail with invalid match regex' do
+      expect { make_filter(:include_regex => '(') }.to raise_error(RuntimeError)
+    end
+
+    it 'should fail with invalid exclude regex' do
+      expect { make_filter(:exclude_regex => '(') }.to raise_error(RuntimeError)
+    end
+  end
+
+  describe '#should_process?' do
+    let(:java_blob) do
+      double('Blob',
+             :getBucket => 'bucket',
+             :getMetadata => { 'some-tag' => 'true' },
+             :getName => 'path/to/log.log.gz',
+             :getMd5 => 'md5',
+             :getCrc32c => 'crc',
+             :getGeneration => 123,
+             :downloadTo => true)
+    end
+    let(:blob) { LogStash::Inputs::CloudStorage::BlobAdapter.new(java_blob) }
+    let(:passdb) { double('passdb', :already_processed? => false) }
+    let(:faildb) { double('passdb', :already_processed? => true) }
+
+    it 'should be able to pass with a blob that meets all conditions' do
+      subject = make_filter
+
+      expect(subject.should_process?(blob)).to eq(true)
+    end
+
+    it 'should fail if a blob is in the processed database' do
+      subject = make_filter(:processed_db => faildb)
+
+      expect(subject.should_process?(blob)).to eq(false)
+    end
+
+    it 'should fail if a blob does not match the include regex' do
+      subject = make_filter(:include_regex => '.*.json')
+
+      expect(subject.should_process?(blob)).to eq(false)
+    end
+
+    it 'should fail if a blob matches the exclude regex' do
+      subject = make_filter(:exclude_regex => '.*')
+
+      expect(subject.should_process?(blob)).to eq(false)
+
+    end
+
+    it 'should fail if it has an already processed metadata tag' do
+      subject = make_filter(:metadata_key => 'some-tag')
+
+      expect(subject.should_process?(blob)).to eq(false)
+
+    end
+  end
+
+  def make_filter(options = {})
+    logger = spy('logger')
+
+    LogStash::Inputs::CloudStorage::BlobFilter.new(
+      logger,
+      options[:include_regex] || '.*',
+      options[:exclude_regex] || '^$',
+      options[:metadata_key] || '',
+      options[:processed_db] || nil
+    )
+  end
+end

data/spec/inputs/cloud_storage/client_spec.rb

@@ -0,0 +1,33 @@
+# encoding: utf-8
+
+# Copyright 2018 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require 'logstash/devutils/rspec/spec_helper'
+require 'logstash/inputs/google_cloud_storage'
+require 'logstash/inputs/cloud_storage/client'
+
+describe LogStash::Inputs::CloudStorage::Client do
+
+  # This test is mostly to make sure the Java types, signatures and classes
+  # haven't changed being that JRuby is very relaxed.
+  describe '#initialize' do
+    let(:logger) { spy('logger') }
+
+    it 'does not throw an error when initializing' do
+      key_file = ::File.join('spec', 'fixtures', 'credentials.json')
+      LogStash::Inputs::CloudStorage::Client.new('my-bucket', key_file, logger)
+    end
+  end
+end

data/spec/inputs/cloud_storage/file_reader_spec.rb

@@ -0,0 +1,76 @@
+# encoding: utf-8
+
+# Copyright 2018 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require 'logstash/devutils/rspec/spec_helper'
+require 'logstash/inputs/google_cloud_storage'
+require 'logstash/inputs/cloud_storage/file_reader'
+
+describe LogStash::Inputs::CloudStorage::FileReader do
+  describe '#gzip?' do
+    GzipTest = Struct.new('GzipTest', :path, :expected)
+
+    it 'is true when the file ends with .gz' do
+      cases = [
+        GzipTest.new('simple.gz', true),
+        GzipTest.new('subdir/simple.gz', true),
+        GzipTest.new('/abs/path.gz', true),
+        GzipTest.new('path.log.gz', true),
+
+        GzipTest.new('path.log.gzip', false),
+        GzipTest.new('simple.log', false),
+        GzipTest.new('simple.gz/foo', false),
+        GzipTest.new('simple.gz.log', false),
+        GzipTest.new('foo.tgz', false)
+      ]
+
+      cases.each do |test|
+        result = LogStash::Inputs::CloudStorage::FileReader.gzip?(test.path)
+
+        expect(result).to eq(test.expected)
+      end
+    end
+  end
+
+  describe '#read_plain_lines' do
+    let(:path) { ::File.join('spec', 'fixtures', 'helloworld.log') }
+
+    it 'reads plain files' do
+      expected = [['hello', 1], ['world', 2]]
+
+      out = []
+      LogStash::Inputs::CloudStorage::FileReader.read_plain_lines(path) do |text, linenum|
+        out << [text, linenum]
+      end
+
+      expect(out).to eq(expected)
+    end
+  end
+
+  describe '#read_gzip_lines' do
+    let(:path) { ::File.join('spec', 'fixtures', 'helloworld.log.gz') }
+
+    it 'reads gzipped files' do
+      expected = [['hello', 1], ['world', 2]]
+
+      out = []
+      LogStash::Inputs::CloudStorage::FileReader.read_gzip_lines(path) do |text, linenum|
+        out << [text, linenum]
+      end
+
+      expect(out).to eq(expected)
+    end
+  end
+end

data/spec/inputs/cloud_storage/processed_db_spec.rb

@@ -0,0 +1,82 @@
+# encoding: utf-8
+
+# Copyright 2018 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require 'logstash/devutils/rspec/spec_helper'
+require 'logstash/inputs/google_cloud_storage'
+require 'logstash/inputs/cloud_storage/processed_db'
+
+describe LogStash::Inputs::CloudStorage::ProcessedDb do
+
+  let(:blob) { double('blob', :name => 'foo', :generation => 12_345) }
+  subject { LogStash::Inputs::CloudStorage::ProcessedDb.new('base') }
+
+  describe '#encode_path' do
+    it 'includes the generation in the hash' do
+      blob2 = double('blob2', :name => 'foo', :generation => 54_321)
+
+      expect(subject.encode_path(blob)).to_not eq(subject.encode_path(blob2))
+    end
+
+    it 'includes the path in the hash' do
+      blob2 = double('blob2', :name => 'bar', :generation => 12_345)
+
+      expect(subject.encode_path(blob)).to_not eq(subject.encode_path(blob2))
+    end
+
+    it 'includes the base directory in the path' do
+      path = subject.encode_path(blob)
+
+      expect(path.start_with?('base')).to eq(true)
+    end
+
+    it 'splits the path into base/3/rest' do
+      path = subject.encode_path(blob)
+      parts = path.split(::File::SEPARATOR)
+
+      expect(parts.length).to eq(3)
+      expect(parts[0]).to eq('base')
+      expect(parts[1].length).to eq(3)
+    end
+
+    it 'produces a correct SHA1 path' do
+      expected = ::File.join('base', 'd27', '628eefc02ae87401aea8c57c49579fbd6b55e')
+
+      expect(subject.encode_path(blob)).to eq(expected)
+    end
+  end
+
+  # The tight coupling of the tests to already_processed? and mark_processed
+  # is intentional.
+  # It's critical the implementation remains the same between versions
+  # so upgrading the plugin version won't trigger a full re-index of a bucket.
+
+  describe '#already_processed?' do
+    it 'checks if the path returned from encode_path exists' do
+      allow(subject).to receive(:encode_path).and_return('path/to/create')
+      expect(::File).to receive(:exist?).with('path/to/create')
+
+      subject.already_processed?(blob)
+    end
+  end
+
+  describe '#mark_processed' do
+    it 'creates the directory and parent dirs for encode_path' do
+      allow(subject).to receive(:encode_path).and_return('path/to/create')
+
+      subject.already_processed?(blob)
+    end
+  end
+end