logstash-core 2.4.0.snapshot1-java → 2.4.0.snapshot2-java

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 55e9a573cfdc0b9517659c97f9f4cdaf2bdc05de
-  data.tar.gz: 32ac5a48ddfa1c73096d87a5a046f0e57b293290
+  metadata.gz: d434dcd578f2d8de2e1e04b8a93b9747431d239d
+  data.tar.gz: a385001b8eb874819f9ba4a5c9c04c88a4e2ea15
 SHA512:
-  metadata.gz: a9a2280fc17d181b68dffe2fe913e7be848b7b4423a762dd852c27862d4c1aa68f9e86d967b8e5dcdb4eafa22cc628d72e3a8dea6e0782d043c08e00ab83aa7b
-  data.tar.gz: b3a25acf25c18efa3a61834b954380c20aff7aca041e3db6cf9556792935799a3e37230d2c0b260217a90e00414839cd088a9b8685306df7ab76a2dec180a2e6
+  metadata.gz: fb9478658883a032ad4f2e779d835d16feab56b4e88880ee9e6b8de4e5ca1a355e315965b5db7a8fe0a6d96cd838849fbcc6ec46ae0e602c745c8bcf0ab38197
+  data.tar.gz: bb1e11a7cb48bcf052298e9031eee04db3ce27c11551c0f2e6a94e35b1b94daf9e5beac65fe58c0f528d1d13fe31ab6a88dd287dc6619c93aa88495867bb6bfe

data/lib/logstash/codecs/base.rb

@@ -13,6 +13,7 @@ module LogStash::Codecs; class Base < LogStash::Plugin
     super
     config_init(@params)
     register if respond_to?(:register)
+    setup_multi_encode!
   end
 
   public
@@ -23,10 +24,37 @@ module LogStash::Codecs; class Base < LogStash::Plugin
   alias_method :<<, :decode
 
   public
+  # DEPRECATED: Prefer defining encode_sync or multi_encode
   def encode(event)
-    raise "#{self.class}#encode must be overidden"
+    encoded = multi_encode([event])
+    encoded.each {|event,data| @on_event.call(event,data) }
   end # def encode
 
+  public
+  # Relies on the codec being synchronous (which they all are!)
+  # We need a better long term design here, but this is an improvement
+  # over the current API for shared plugins
+  # It is best if the codec implements this directly
+  def multi_encode(events)
+    if @has_encode_sync              
+      events.map {|event| [event, self.encode_sync(event)]}
+    else
+      batch = Thread.current[:logstash_output_codec_batch] ||= []
+      batch.clear
+      
+      events.each {|event| self.encode(event) }
+      batch
+    end
+  end
+
+  def setup_multi_encode!
+    @has_encode_sync = self.methods.include?(:encode_sync)
+
+    on_event do |event, data|
+      Thread.current[:logstash_output_codec_batch] << [event, data]
+    end
+  end
+
   public
   def close; end;
 

data/lib/logstash/outputs/base.rb

@@ -27,6 +27,22 @@ class LogStash::Outputs::Base < LogStash::Plugin
 
   attr_reader :worker_plugins, :available_workers, :workers, :worker_plugins, :workers_not_supported
 
+    # Set or return concurrency type
+  def self.concurrency(type=nil)
+    if type
+      @concurrency = type
+      
+      if type == :shared
+        declare_threadsafe!
+      elsif type == :single
+        declare_workers_not_supported!("This plugin only supports one worker!")
+      end
+      
+    else
+      @concurrency || :legacy # default is :legacyo
+    end
+  end
+
   def self.declare_threadsafe!
     declare_workers_not_supported!
     @threadsafe = true
@@ -65,6 +81,8 @@ class LogStash::Outputs::Base < LogStash::Plugin
     # If we're running with a single thread we must enforce single-threaded concurrency by default
     # Maybe in a future version we'll assume output plugins are threadsafe
     @single_worker_mutex = Mutex.new
+    
+    @receives_encoded = self.methods.include?(:multi_receive_encoded)
   end
 
   public
@@ -77,10 +95,23 @@ class LogStash::Outputs::Base < LogStash::Plugin
     raise "#{self.class}#receive must be overidden"
   end # def receive
 
+  public
+  def concurrency
+    self.class.concurrency
+  end
+
   public
   # To be overriden in implementations
   def multi_receive(events)
-    events.each {|event| receive(event) }
+    if @receives_encoded
+      self.multi_receive_encoded(codec.multi_encode(events))
+    else
+      events.each {|event| receive(event) }
+    end
+  end
+
+  def codec
+    params["codec"]
   end
 
   private

data/lib/logstash/version.rb

@@ -11,4 +11,4 @@
 #       eventually this file should be in the root logstash lib fir and dependencies in logstash-core should be
 #       fixed.
 
-LOGSTASH_VERSION = "2.4.0.snapshot1"
+LOGSTASH_VERSION = "2.4.0.snapshot2"

data/lib/logstash-core/version.rb

@@ -5,4 +5,4 @@
 # Note to authors: this should not include dashes because 'gem' barfs if
 # you include a dash in the version string.
 
-LOGSTASH_CORE_VERSION = "2.4.0.snapshot1"
+LOGSTASH_CORE_VERSION = "2.4.0.snapshot2"

data/logstash-core.gemspec

@@ -17,7 +17,7 @@ Gem::Specification.new do |gem|
   gem.require_paths = ["lib"]
   gem.version       = LOGSTASH_CORE_VERSION
 
-  gem.add_runtime_dependency "logstash-core-event", "2.4.0.snapshot1"
+  gem.add_runtime_dependency "logstash-core-event", "2.4.0.snapshot2"
   # gem.add_runtime_dependency "logstash-core-event-java", "2.4.0.dev"
 
   gem.add_runtime_dependency "cabin", "~> 0.8.0" #(Apache 2.0 license)

data/spec/logstash/codecs/base_spec.rb

@@ -0,0 +1,74 @@
+# encoding: utf-8
+require "spec_helper"
+
+DATA_DOUBLE = "data".freeze
+
+# use a dummy NOOP output to test Outputs::Base
+class LogStash::Codecs::NOOPAsync < LogStash::Codecs::Base
+  attr_reader :last_result
+  config_name "noop_async"
+
+  def encode(event)
+    @last_result = @on_event.call(event, DATA_DOUBLE)
+  end
+end
+
+class LogStash::Codecs::NOOPSync < LogStash::Codecs::Base
+  attr_reader :last_result
+  config_name "noop_sync"
+
+  def encode_sync(event)
+    DATA_DOUBLE
+  end
+end
+
+class LogStash::Codecs::NOOPMulti < LogStash::Codecs::Base
+  attr_reader :last_result
+  config_name "noop_multi"
+
+  def encode_sync(event)
+    DATA_DOUBLE
+  end
+end
+
+describe LogStash::Codecs::Base do
+  let(:params) { {} }
+  subject(:instance) { klass.new(params.dup) }
+  let(:event) { double("event") }
+  let(:encoded_data) { DATA_DOUBLE }
+  let(:encoded_tuple) { [event, encoded_data] }
+
+  describe "encoding" do
+    shared_examples "encoder types" do |codec_class|
+      let(:klass) { codec_class }
+      
+      describe "#{codec_class}" do
+        describe "multi_encode" do
+          it "should return an array of [event,data] tuples" do
+            expect(instance.multi_encode([event,event])).to eq([encoded_tuple, encoded_tuple])
+          end
+        end
+        
+        describe "#encode" do
+          before do
+            @result = nil
+            instance.on_event do |event, data|
+              @result = [event, data]
+            end
+            instance.encode(event)
+          end
+          
+          it "should yield the correct result" do
+            expect(@result).to eq(encoded_tuple)
+          end
+        end
+      end
+    end
+
+    include_examples("encoder types", LogStash::Codecs::NOOPAsync)
+    include_examples("encoder types", LogStash::Codecs::NOOPSync)
+    include_examples("encoder types", LogStash::Codecs::NOOPMulti)
+  end
+end
+
+                          

data/spec/logstash/outputs/base_spec.rb

@@ -15,6 +15,31 @@ class LogStash::Outputs::NOOP < LogStash::Outputs::Base
   end
 end
 
+
+# use a dummy NOOP output to test Outputs::Base
+class LogStash::Outputs::NOOPSingle < LogStash::Outputs::Base
+  config_name "noop single"
+  concurrency :single
+
+  config :dummy_option, :validate => :string
+
+  def register; end
+
+  def receive(event)
+    return output?(event)
+  end
+end
+
+class LogStash::Outputs::NOOPShared < ::LogStash::Outputs::Base
+  concurrency :shared
+  
+  def register; end
+end
+
+class LogStash::Outputs::NOOPLegacy < ::LogStash::Outputs::Base
+  def register; end
+end
+
 class LogStash::Outputs::NOOPLegacyNoWorkers < ::LogStash::Outputs::Base
   LEGACY_WORKERS_NOT_SUPPORTED_REASON = "legacy reason"
 
@@ -23,7 +48,52 @@ class LogStash::Outputs::NOOPLegacyNoWorkers < ::LogStash::Outputs::Base
   end
 end
 
+class LogStash::Outputs::NOOPMultiReceiveEncoded < ::LogStash::Outputs::Base
+  concurrency :single
+  
+  def register; end
+
+  def multi_receive_encoded(events_and_encoded)
+  end
+end
+
 describe "LogStash::Outputs::Base#new" do
+  describe "concurrency" do
+    subject { klass.new({}) }
+    
+    context "single" do   
+      let(:klass) { LogStash::Outputs::NOOPSingle }
+
+      it "should set concurrency correctly" do
+        expect(subject.concurrency).to eq(:single)
+      end
+    end
+
+    context "shared" do
+      let(:klass) { LogStash::Outputs::NOOPShared }
+      
+      it "should set concurrency correctly" do
+        expect(subject.concurrency).to eq(:shared)
+      end
+    end
+
+    context "legacy" do
+      let(:klass) { LogStash::Outputs::NOOPLegacy }
+      
+      it "should set concurrency correctly" do
+        expect(subject.concurrency).to eq(:legacy)
+      end
+
+      it "should default the # of workers to 1" do
+        expect(subject.workers).to eq(1)
+      end
+
+      it "should default concurrency to :legacy" do
+        expect(subject.concurrency).to eq(:legacy)
+      end
+    end
+  end
+  
   it "should instantiate cleanly" do
     params = { "dummy_option" => "potatoes", "codec" => "json", "workers" => 2 }
     worker_params = params.dup; worker_params["workers"] = 1
@@ -37,4 +107,41 @@ describe "LogStash::Outputs::Base#new" do
     LogStash::Outputs::NOOPLegacyNoWorkers.new.register
     expect(LogStash::Outputs::NOOPLegacyNoWorkers.workers_not_supported?).to eql(true)
   end
+
+  describe "dispatching multi_receive" do
+    let(:event) { double("event") }
+    let(:events) { [event] }
+    subject { klass.new({}) }
+    
+    context "with multi_receive_encoded" do
+      let(:klass) { LogStash::Outputs::NOOPMultiReceiveEncoded }
+      let(:codec) { double("codec") }
+      let(:encoded) { double("encoded") }
+      
+      before do
+        allow(codec).to receive(:multi_encode).with(events).and_return(encoded)
+        allow(subject).to receive(:codec).and_return(codec)
+        allow(subject).to receive(:multi_receive_encoded)
+        subject.multi_receive(events)
+      end
+
+      it "should invoke multi_receive_encoded if it exists" do
+        expect(subject).to have_received(:multi_receive_encoded).with(encoded)
+      end
+    end
+
+    context "with plain #receive" do
+      let(:klass) { LogStash::Outputs::NOOPSingle }
+
+      before do
+        allow(subject).to receive(:multi_receive).and_call_original
+        allow(subject).to receive(:receive).with(event)
+        subject.multi_receive(events)
+      end
+
+      it "should receive the event by itself" do
+        expect(subject).to have_received(:receive).with(event)
+      end
+    end
+  end
 end

metadata

@@ -1,21 +1,21 @@
 --- !ruby/object:Gem::Specification
 name: logstash-core
 version: !ruby/object:Gem::Version
-  version: 2.4.0.snapshot1
+  version: 2.4.0.snapshot2
 platform: java
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2016-08-19 00:00:00.000000000 Z
+date: 2016-08-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 2.4.0.snapshot1
+        version: 2.4.0.snapshot2
   name: logstash-core-event
   prerelease: false
   type: :runtime
@@ -23,7 +23,7 @@ dependencies:
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 2.4.0.snapshot1
+        version: 2.4.0.snapshot2
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
@@ -287,6 +287,7 @@ files:
 - logstash-core.gemspec
 - spec/conditionals_spec.rb
 - spec/logstash/agent_spec.rb
+- spec/logstash/codecs/base_spec.rb
 - spec/logstash/config/config_ast_spec.rb
 - spec/logstash/config/cpu_core_strategy_spec.rb
 - spec/logstash/config/defaults_spec.rb
@@ -343,6 +344,7 @@ summary: logstash-core - The core components of logstash
 test_files:
 - spec/conditionals_spec.rb
 - spec/logstash/agent_spec.rb
+- spec/logstash/codecs/base_spec.rb
 - spec/logstash/config/config_ast_spec.rb
 - spec/logstash/config/cpu_core_strategy_spec.rb
 - spec/logstash/config/defaults_spec.rb