logstash-output-google_bigquery 4.0.0-java

data/logstash-output-google_bigquery.gemspec

@@ -0,0 +1,31 @@
+Gem::Specification.new do |s|
+  s.name            = 'logstash-output-google_bigquery'
+  s.version         = '4.0.0'
+  s.licenses        = ['Apache License (2.0)']
+  s.summary         = "Writes events to Google BigQuery"
+  s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
+  s.authors         = ["Elastic"]
+  s.email           = 'info@elastic.co'
+  s.homepage        = "http://www.elastic.co/guide/en/logstash/current/index.html"
+  s.require_paths   = ["lib", "vendor/jar-dependencies"]
+
+  # Files
+  s.files = Dir["lib/**/*","spec/**/*","*.gemspec","*.md","CONTRIBUTORS","Gemfile","LICENSE","NOTICE.TXT", "vendor/jar-dependencies/**/*.jar", "vendor/jar-dependencies/**/*.rb", "VERSION", "docs/**/*"]
+
+  # Tests
+  s.test_files = s.files.grep(%r{^(test|spec|features)/})
+
+  # Special flag to let us know this is actually a logstash plugin
+  s.metadata = { "logstash_plugin" => "true", "logstash_group" => "output" }
+
+  # Gem dependencies
+  s.add_runtime_dependency 'logstash-codec-plain'
+  s.add_runtime_dependency 'mime-types', '~> 2' # last version compatible with ruby 2.x
+  s.add_runtime_dependency "logstash-core-plugin-api", ">= 1.60", "<= 2.99"
+
+  s.add_development_dependency 'logstash-devutils'
+  s.add_development_dependency 'jar-dependencies', '~> 0.3.4'
+
+  s.platform = 'java'
+end
+

data/spec/outputs/bigquery/batcher_spec.rb

@@ -0,0 +1,110 @@
+require 'logstash/outputs/bigquery/batcher'
+require 'thread'
+
+describe LogStash::Outputs::BigQuery::Batcher do
+  let(:logger) { spy(:logger) }
+  let(:batcher) { LogStash::Outputs::BigQuery::Batcher.new(2, 1_000) }
+  let(:one_b_message) { 'a' }
+  let(:one_k_message) { 'a' * 1000 }
+  let(:batch_queue) { Queue.new }
+
+  describe '#enqueue' do
+    it 'returns nil if no reason to flush' do
+      batch = batcher.enqueue one_b_message
+
+      expect(batch).to be_nil
+    end
+
+    it 'returns a batch if passed nil' do
+      batch = batcher.enqueue nil
+
+      expect(batch).to_not be_nil
+    end
+
+    it 'returns a batch if the message count overflows' do
+      batch = batcher.enqueue one_b_message
+      expect(batch).to be_nil
+
+      batch = batcher.enqueue one_b_message
+      expect(batch).to_not be_nil
+    end
+
+    it 'returns a batch if the message size overflows' do
+      batch = batcher.enqueue one_b_message
+      expect(batch).to be_nil
+
+      batch = batcher.enqueue one_k_message
+      expect(batch).to_not be_nil
+    end
+
+    it 'clears internal state after returning a batch' do
+      batch = batcher.enqueue one_k_message
+
+      expect(batch).to_not be_nil
+      expect(batcher.empty?).to be_truthy
+    end
+
+    it 'does not yield a batch if there is no reason to flush' do
+      batch = nil
+      batcher.enqueue(one_b_message) { |b| batch = b }
+
+      expect(batch).to be_nil
+    end
+
+    it 'yields a batch on flush' do
+      batch = nil
+      batcher.enqueue(nil) { |b| batch = b }
+
+      expect(batch).to_not be_nil
+      expect(batch.length).to eq 0
+    end
+
+    it 'yields a batch on overflow' do
+      batch = nil
+      batcher.enqueue(one_k_message) { |b| batch = b }
+
+      expect(batch).to_not be_nil
+      expect(batch.length).to eq 1
+    end
+  end
+
+  describe '#enqueue_push' do
+    it 'enqueues nothing nil if no reason to flush' do
+      batcher.enqueue_push one_b_message, batch_queue
+
+      expect(batch_queue.length).to eq 0
+    end
+
+    it 'enqueues a batch if passed nil' do
+      batcher.enqueue_push nil, batch_queue
+
+      expect(batch_queue.length).to eq 1
+    end
+
+    it 'enqueues a batch if the message count overflows' do
+      batcher.enqueue_push one_b_message, batch_queue
+      expect(batch_queue.length).to eq 0
+
+      batcher.enqueue_push one_b_message, batch_queue
+      expect(batch_queue.length).to eq 1
+    end
+
+    it 'enqueues a batch if the message size overflows' do
+      batcher.enqueue_push one_b_message, batch_queue
+      expect(batch_queue.length).to eq 0
+
+      batcher.enqueue_push one_k_message, batch_queue
+      expect(batch_queue.length).to eq 1
+    end
+  end
+
+  describe '#clear' do
+    it 'removes any existing messages' do
+      batcher.enqueue one_b_message
+      expect(batcher.empty?).to be_falsey
+
+      batcher.clear
+      expect(batcher.empty?).to be_truthy
+    end
+  end
+end

data/spec/outputs/bigquery/schema_spec.rb

@@ -0,0 +1,101 @@
+require 'logstash/devutils/rspec/spec_helper'
+require 'logstash/outputs/bigquery/schema'
+require 'thread'
+require 'java'
+
+describe LogStash::Outputs::BigQuery::Schema do
+
+  let(:simple_field) {keys_to_strs({name: 'foo', type: 'STRING'})}
+  let(:complex_field) {keys_to_strs({name: 'params', type: 'RECORD', mode: 'REPEATED', description: 'desc', fields: [simple_field]})}
+  let(:example_field_list) {[simple_field, complex_field]}
+  let(:example_schema) {keys_to_strs({fields:example_field_list})}
+
+  describe '#parse_csv_or_json' do
+    it 'ensures CSV and JSON are not both nil' do
+      expect{LogStash::Outputs::BigQuery::Schema.parse_csv_or_json nil, nil}.to raise_error ArgumentError
+    end
+
+    it 'ensures CSV and JSON are not both defined' do
+      expect{LogStash::Outputs::BigQuery::Schema.parse_csv_or_json "", {}}.to raise_error ArgumentError
+    end
+
+    it 'parses a CSV schema if it exists' do
+      result = LogStash::Outputs::BigQuery::Schema.parse_csv_or_json "name:STRING", nil
+      expect(result).to_not be_nil
+    end
+
+    it 'converts the resulting schema into a Java one' do
+      result = LogStash::Outputs::BigQuery::Schema.parse_csv_or_json "name:STRING", nil
+      expect(result.getClass().getName()).to eq('com.google.cloud.bigquery.Schema')
+    end
+  end
+
+  describe '#parse_csv_schema' do
+    it 'splits a CSV into name->type structures' do
+      expected = {fields: [keys_to_strs({name:'foo', type:'STRING'}), keys_to_strs({name:'bar', type:'FLOAT'})]}
+      result = LogStash::Outputs::BigQuery::Schema.parse_csv_schema "foo:STRING,bar:FLOAT"
+
+      expect(result).to eq(keys_to_strs(expected))
+    end
+
+    it 'fails on a malformed CSV' do
+      expect{LogStash::Outputs::BigQuery::Schema.parse_csv_schema "foo:bar:bazz"}.to raise_error ArgumentError
+      expect{LogStash::Outputs::BigQuery::Schema.parse_csv_schema "foo:bar,,bar:bazz"}.to raise_error ArgumentError
+      expect{LogStash::Outputs::BigQuery::Schema.parse_csv_schema "foo:bar,"}.to raise_error ArgumentError
+    end
+  end
+
+  describe '#hash_to_java_schema' do
+    subject{LogStash::Outputs::BigQuery::Schema.hash_to_java_schema(example_schema)}
+
+    it 'parses the field list from the fields key' do
+      expect(subject.getFields().size()).to eq(2)
+    end
+
+    it 'returns a BigQuery Schema object' do
+      expect(subject.getClass().getName()).to eq('com.google.cloud.bigquery.Schema')
+    end
+  end
+
+  describe '#parse_field_list' do
+    subject{LogStash::Outputs::BigQuery::Schema.parse_field_list(example_field_list)}
+
+    it 'returns a Java FieldList object' do
+      expect(subject.getClass().getName()).to eq('com.google.cloud.bigquery.FieldList')
+    end
+  end
+
+  describe '#parse_field' do
+    subject{LogStash::Outputs::BigQuery::Schema.parse_field(complex_field)}
+
+    it 'sets the correct name and type' do
+      expect(subject.getName()).to eq('params')
+      expect(subject.getType().toString()).to eq('RECORD')
+    end
+
+    it 'sets a description and mode if present' do
+      expect(subject.getDescription()).to eq('desc')
+    end
+    
+    it 'sets sub-fields if present' do
+      expect(subject.getSubFields().size()).to eq(1)
+    end
+
+    it 'returns a Java Field object' do
+      expect(subject.getClass().getName()).to eq('com.google.cloud.bigquery.Field')
+    end
+  end
+
+  # converts tokens into strings recursively for a map.
+  def keys_to_strs(event)
+    return event unless event.is_a? Hash
+
+    out = {}
+
+    event.each do |key, value|
+      out[key.to_s] = keys_to_strs value
+    end
+
+    out
+  end
+end

data/spec/outputs/google_bigquery_spec.rb

@@ -0,0 +1,154 @@
+require "logstash/devutils/rspec/spec_helper"
+require 'logstash/outputs/google_bigquery'
+require 'logstash/outputs/bigquery/streamclient'
+
+describe LogStash::Outputs::GoogleBigQuery do
+
+  let(:config) { { 'project_id' => 'project', 'dataset' => 'dataset', 'csv_schema' => 'path:STRING,status:INTEGER,score:FLOAT' } }
+  let(:sample_event) { LogStash::Event.new }
+  let(:bq_client) { double('streaming-client') }
+  let(:errors_file) { double('errors file') }
+
+  subject { LogStash::Outputs::GoogleBigQuery.new(config) }
+
+  before(:each) do
+    allow(LogStash::Outputs::BigQuery::StreamingClient).to receive(:new).and_return(bq_client)
+    expect(LogStash::Outputs::BigQuery::StreamingClient).to receive(:new)
+
+    allow(subject).to receive(:init_batcher_flush_thread).and_return(nil)
+    expect(subject).to receive(:init_batcher_flush_thread)
+
+    subject.register
+
+  end
+
+  describe '#get_table_name' do
+    it 'does not crash if no time is given' do
+      subject.get_table_name
+    end
+
+    it 'formats the table name correctly' do
+      table_id = subject.get_table_name Time.new(2012,9,8,7,6)
+      expect(table_id).to eq('logstash_2012_09_08T07_00')
+    end
+  end
+
+  describe '#replace_at_keys' do
+    it 'removes @ in keys' do
+      nested = {'@foo' => 'bar'}
+      expected = {foo: 'bar'}
+
+      out = subject.replace_at_keys nested
+
+      expect(out).to eq(keys_to_strs(expected))
+    end
+
+    it 'does not remove @ in values' do
+      nested = {foo: '@bar'}
+
+      out = subject.replace_at_keys nested
+
+      expect(out).to eq(keys_to_strs(nested))
+
+    end
+
+    it 'removes @ in nested keys' do
+      nested = {foo: {'@bar' => 'bazz'}}
+      expected = {foo: {bar: 'bazz'}}
+
+      out = subject.replace_at_keys nested
+
+      expect(out).to eq(keys_to_strs(expected))
+    end
+  end
+
+  describe '#publish' do
+    it 'does nothing if there are no messages' do
+      allow(subject).to receive(:create_table_if_not_exists).and_return(nil)
+
+      subject.publish nil
+      subject.publish []
+
+      expect(subject).not_to receive(:create_table_if_not_exists)
+    end
+
+    it 'creates a table if it does not exist' do
+      allow(subject).to receive(:create_table_if_not_exists).and_return(nil)
+      allow(bq_client).to receive(:append).and_return(true)
+      allow(subject).to receive(:write_to_errors_file).and_return(nil)
+      expect(subject).to receive(:create_table_if_not_exists)
+
+      subject.publish ['{"foo":"bar"}']
+    end
+
+    it 'writes rows to a file on failed insert' do
+      allow(subject).to receive(:create_table_if_not_exists).and_return(nil)
+      allow(bq_client).to receive(:append).and_return(false)
+      allow(subject).to receive(:write_to_errors_file).and_return(nil)
+      expect(subject).to receive(:write_to_errors_file)
+
+      subject.publish ['{"foo":"bar"}']
+    end
+
+    it 'writes rows to a file if insert threw an exception' do
+      allow(subject).to receive(:create_table_if_not_exists).and_return(nil)
+      allow(bq_client).to receive(:append).and_raise('expected insert error')
+      allow(subject).to receive(:write_to_errors_file).and_return(nil)
+      expect(subject).to receive(:write_to_errors_file)
+
+      subject.publish ['{"foo":"bar"}']
+    end
+  end
+
+  describe '#create_table_if_not_exists' do
+    it 'checks if a table exists' do
+      allow(bq_client).to receive(:table_exists?).and_return(true)
+      expect(bq_client).to receive(:table_exists?)
+
+      subject.create_table_if_not_exists 'foo'
+    end
+
+    it 'creates a table if it does not exist' do
+      allow(bq_client).to receive(:table_exists?).and_return(false)
+      allow(bq_client).to receive(:create_table).and_return(nil)
+      expect(bq_client).to receive(:table_exists?)
+      expect(bq_client).to receive(:create_table)
+
+      subject.create_table_if_not_exists 'foo'
+    end
+  end
+
+  describe '#write_to_errors_file' do
+      it 'creates missing directories' do
+        allow(File).to receive(:open).and_return(errors_file)
+        allow(FileUtils).to receive(:mkdir_p)
+        expect(FileUtils).to receive(:mkdir_p)
+
+        subject.write_to_errors_file(['a','b'], 'table_name')
+      end
+
+      it 'does not fail on exception' do
+        allow(FileUtils).to receive(:mkdir_p).and_raise("exception creating directories")
+        expect{subject.write_to_errors_file([], 'table_name')}.to_not raise_error
+      end
+  end
+
+
+  # converts tokens into strings recursively for a map.
+  def keys_to_strs(event)
+    return event unless event.is_a? Hash
+
+    out = {}
+
+    event.each do |key, value|
+      out[key.to_s] = keys_to_strs value
+    end
+
+    out
+  end
+
+  RSpec::Matchers.define :starts_with do |x|
+    match { |actual| actual.start_with? x}
+  end
+
+end