logstash-core 6.3.2-java → 6.4.0-java

data/lib/logstash/logging/json.rb

@@ -1,21 +0,0 @@
-# encoding: utf-8
-require "logstash/namespace"
-require "logstash/logging"
-require "logstash/json"
-
-module LogStash; module Logging; class JSON
-  def initialize(io)
-    raise ArgumentError, "Expected IO, got #{io.class.name}" unless io.is_a?(IO)
-
-    @io = io
-    @lock = Mutex.new
-  end
-
-  def <<(obj)
-    serialized = LogStash::Json.dump(obj)
-    @lock.synchronize do
-      @io.puts(serialized)
-      @io.flush
-    end
-  end
-end; end; end

data/lib/logstash/plugins/plugin_factory.rb

@@ -1,107 +0,0 @@
-# encoding: utf-8
-
-module LogStash
-  module Plugins
-
-    class ExecutionContextFactory
-
-      def initialize(agent, pipeline, dlq_writer)
-        @agent = agent
-        @pipeline = pipeline
-        @dlq_writer = dlq_writer
-      end
-
-      def create(id, klass_cfg_name)
-        ExecutionContext.new(@pipeline, @agent, id, klass_cfg_name, @dlq_writer)
-      end
-    end
-
-    class PluginMetricFactory
-
-      def initialize(pipeline_id, metric)
-        @pipeline_id = pipeline_id.to_s.to_sym
-        @metric = metric
-      end
-
-      def create(plugin_type)
-        @metric.namespace([:stats, :pipelines, @pipeline_id, :plugins])
-          .namespace("#{plugin_type}s".to_sym)
-      end
-    end
-
-    class PluginFactory
-      include org.logstash.config.ir.compiler.RubyIntegration::PluginFactory
-
-      def self.filter_delegator(wrapper_class, filter_class, args, filter_metrics, execution_context)
-        filter_instance = filter_class.new(args)
-        id = args["id"]
-        filter_instance.metric = filter_metrics.namespace(id.to_sym)
-        filter_instance.execution_context = execution_context
-        wrapper_class.new(filter_instance, id)
-      end
-
-      def initialize(lir, metric_factory, exec_factory, filter_class)
-        @lir = lir
-        @plugins_by_id = {}
-        @metric_factory = metric_factory
-        @exec_factory = exec_factory
-        @filter_class = filter_class
-      end
-
-      def buildOutput(name, line, column, *args)
-        plugin("output", name, line, column, *args)
-      end
-
-      def buildFilter(name, line, column, *args)
-        plugin("filter", name, line, column, *args)
-      end
-
-      def buildInput(name, line, column, *args)
-        plugin("input", name, line, column, *args)
-      end
-
-      def buildCodec(name, *args)
-        plugin("codec", name, 0, 0, *args)
-      end
-
-      def plugin(plugin_type, name, line, column, *args)
-        # Collapse the array of arguments into a single merged hash
-        args = args.reduce({}, &:merge)
-
-        if plugin_type == "codec"
-          id = SecureRandom.uuid # codecs don't really use their IDs for metrics, so we can use anything here
-        else
-          # Pull the ID from LIR to keep IDs consistent between the two representations
-          id = @lir.graph.vertices.filter do |v|
-            v.source_with_metadata &&
-              v.source_with_metadata.line == line &&
-              v.source_with_metadata.column == column
-          end.findFirst.get.id
-        end
-        args["id"] = id # some code pulls the id out of the args
-
-        raise ConfigurationError, "Could not determine ID for #{plugin_type}/#{plugin_name}" unless id
-        raise ConfigurationError, "Two plugins have the id '#{id}', please fix this conflict" if @plugins_by_id[id]
-
-        @plugins_by_id[id] = true
-        # Scope plugins of type 'input' to 'inputs'
-        type_scoped_metric = @metric_factory.create(plugin_type)
-        klass = Plugin.lookup(plugin_type, name)
-        execution_context = @exec_factory.create(id, klass.config_name)
-
-        if plugin_type == "output"
-          OutputDelegator.new(klass, type_scoped_metric, execution_context, OutputDelegatorStrategyRegistry.instance, args)
-        elsif plugin_type == "filter"
-          self.class.filter_delegator(@filter_class, klass, args, type_scoped_metric, execution_context)
-        else # input or codec plugin
-          plugin_instance = klass.new(args)
-          scoped_metric = type_scoped_metric.namespace(id.to_sym)
-          scoped_metric.gauge(:name, plugin_instance.config_name)
-          plugin_instance.metric = scoped_metric
-          plugin_instance.execution_context = execution_context
-          plugin_instance
-        end
-      end
-    end
-  end
-end

data/lib/logstash/queue_factory.rb

@@ -1,34 +0,0 @@
-# encoding: utf-8
-require "fileutils"
-require "logstash/event"
-require "logstash/namespace"
-
-module LogStash
-  class QueueFactory
-    def self.create(settings)
-      queue_type = settings.get("queue.type")
-      queue_page_capacity = settings.get("queue.page_capacity")
-      queue_max_bytes = settings.get("queue.max_bytes")
-      queue_max_events = settings.get("queue.max_events")
-      checkpoint_max_acks = settings.get("queue.checkpoint.acks")
-      checkpoint_max_writes = settings.get("queue.checkpoint.writes")
-      checkpoint_max_interval = settings.get("queue.checkpoint.interval")
-
-      queue_path = ::File.join(settings.get("path.queue"), settings.get("pipeline.id"))
-
-      case queue_type
-      when "persisted"
-        # persisted is the disk based acked queue
-        FileUtils.mkdir_p(queue_path)
-        LogStash::WrappedAckedQueue.new(queue_path, queue_page_capacity, queue_max_events, checkpoint_max_writes, checkpoint_max_acks, checkpoint_max_interval, queue_max_bytes)
-      when "memory"
-        # memory is the legacy and default setting
-        LogStash::WrappedSynchronousQueue.new(
-          settings.get("pipeline.batch.size") * settings.get("pipeline.workers") * 2
-        )
-      else
-        raise ConfigurationError, "Invalid setting `#{queue_type}` for `queue.type`, supported types are: 'memory' or 'persisted'"
-      end
-    end
-  end
-end

data/lib/logstash/util/retryable.rb

@@ -1,40 +0,0 @@
-# encoding: utf-8
-module LogStash
-  module Retryable
-    # execute retryable code block
-    # @param [Hash] options retryable options
-    # @option options [Fixnum] :tries retries to perform, default 1, set to 0 for infinite retries. 1 means that upon exception the block will be retried once
-    # @option options [Fixnum] :base_sleep seconds to sleep on first retry, default 1
-    # @option options [Fixnum] :max_sleep max seconds to sleep upon exponential backoff, default 1
-    # @option options [Exception] :rescue exception class list to retry on, defaults is Exception, which retries on any Exception.
-    # @option options [Proc] :on_retry call the given Proc/lambda before each retry with the raised exception as parameter
-    def retryable(options = {}, &block)
-      options = {
-        :tries => 1,
-        :rescue => Exception,
-        :on_retry => nil,
-        :base_sleep => 1,
-        :max_sleep => 1,
-      }.merge(options)
-
-      rescue_classes = Array(options[:rescue])
-      max_sleep_retry = Math.log2(options[:max_sleep] / options[:base_sleep])
-      retry_count = 0
-
-      begin
-        return yield(retry_count)
-      rescue *rescue_classes => e
-        raise e if options[:tries] > 0 && retry_count >= options[:tries]
-
-        options[:on_retry].call(retry_count + 1, e) if options[:on_retry]
-
-        # dont compute and maybe overflow exponent on too big a retry count
-        seconds = retry_count < max_sleep_retry ? options[:base_sleep] * (2 ** retry_count) : options[:max_sleep]
-        sleep(seconds)
-
-        retry_count += 1
-        retry
-      end
-    end
-  end
-end

data/spec/logstash/output_delegator_spec.rb

@@ -1,201 +0,0 @@
-# encoding: utf-8
-require "logstash/execution_context"
-require "spec_helper"
-require "support/shared_contexts"
-
-describe LogStash::OutputDelegator do
-
-  let(:events) { 7.times.map { LogStash::Event.new }}
-  let(:plugin_args) { {"id" => "foo", "arg1" => "val1"} }
-  let(:metric) {
-    LogStash::Instrument::NamespacedMetric.new(
-      LogStash::Instrument::Metric.new(LogStash::Instrument::Collector.new), [:output]
-    )
-  }
-  let(:counter_in) {
-    counter = metric.counter(:in)
-    counter.increment(0)
-    counter
-  }
-  let(:counter_out) {
-    counter = metric.counter(:out)
-    counter.increment(0)
-    counter
-  }
-  let(:counter_time) {
-    counter = metric.counter(:duration_in_millis)
-    counter.increment(0)
-    counter
-  }
-
-  include_context "execution_context"
-
-  class FakeOutClass
-
-    def self.set_out_strategy(out_strategy)
-      @@out_strategy = out_strategy
-    end
-
-    def self.set_out_inst(out_inst)
-      @@out_inst = out_inst
-    end
-
-    def self.name
-      "example"
-    end
-
-    def self.concurrency
-      @@out_strategy
-    end
-
-    def self.config_name
-      "dummy_plugin"
-    end
-
-    class << self
-      def new(args)
-        if args == {"id" => "foo", "arg1" => "val1"}
-          @@out_inst
-        else
-          raise "unexpected plugin arguments"
-        end
-      end
-    end
-  end
-
-  let(:out_klass) {FakeOutClass}
-
-  subject { described_class.new(out_klass, metric, execution_context, ::LogStash::OutputDelegatorStrategyRegistry.instance, plugin_args) }
-
-  context "with a plain output plugin" do
-    let(:out_inst) { double("output instance") }
-
-    before(:each) do
-      # use the same metric instance
-      allow(out_inst).to receive(:register)
-      allow(out_inst).to receive(:multi_receive)
-      allow(out_inst).to receive(:metric=).with(any_args)
-      allow(out_inst).to receive(:execution_context=).with(execution_context)
-      allow(out_inst).to receive(:id).and_return("a-simple-plugin")
-      FakeOutClass.set_out_inst(out_inst)
-      FakeOutClass.set_out_strategy(:single)
-    end
-
-    it "should initialize cleanly" do
-      expect { subject }.not_to raise_error
-    end
-
-    it "should push the name of the plugin to the metric" do
-      described_class.new(out_klass, metric, execution_context, ::LogStash::OutputDelegatorStrategyRegistry.instance, plugin_args)
-      expect(metric.collector.snapshot_metric.metric_store.get_with_path("output/foo")[:output][:foo][:name].value).to eq(out_klass.config_name)
-    end
-
-    context "after having received a batch of events" do
-      before do
-        subject.register
-      end
-
-      it "should pass the events through" do
-        expect(out_inst).to receive(:multi_receive).with(events)
-        subject.multi_receive(events)
-      end
-
-      it "should increment the number of events received" do
-        subject.multi_receive(events)
-        store = metric.collector.snapshot_metric.metric_store.get_with_path("output/foo")[:output][:foo][:events]
-        number_of_events = events.length
-        expect(store[:in].value).to eq(number_of_events)
-        expect(store[:out].value).to eq(number_of_events)
-      end
-
-      it "should record the `duration_in_millis`" do
-        value = 0
-        while value == 0
-          subject.multi_receive(events)
-          store = metric.collector.snapshot_metric.metric_store.get_with_path("output/foo")[:output][:foo][:events]
-          value = store[:duration_in_millis].value
-        end
-        expect(value).to be > 0
-      end
-    end
-
-    describe "closing" do
-      before do
-        subject.register
-      end
-
-      it "should register the output plugin instance on register" do
-        expect(out_inst).to have_received(:register)
-      end
-
-      it "should close the output plugin instance when closing" do
-        expect(out_inst).to receive(:do_close)
-        subject.do_close
-      end
-    end
-
-    describe "concurrency strategies" do
-      it "should have :single as the default" do
-        expect(subject.concurrency).to eq :single
-      end
-
-      [
-        [:shared, ::LogStash::OutputDelegatorStrategies::Shared],
-        [:single, ::LogStash::OutputDelegatorStrategies::Single],
-        [:legacy, ::LogStash::OutputDelegatorStrategies::Legacy],
-      ].each do |strategy_concurrency,klass|
-        context "with strategy #{strategy_concurrency}" do
-          let(:concurrency) { strategy_concurrency }
-
-          before(:each) do
-            FakeOutClass.set_out_strategy(strategy_concurrency)
-          end
-
-          it "should find the correct concurrency type for the output" do
-            expect(subject.concurrency).to eq(strategy_concurrency)
-          end
-
-          it "should find the correct Strategy class for the worker" do
-            expect(subject.strategy).to be_a(klass)
-          end
-
-          it "should set the metric on the instance" do
-            expect(out_inst).to have_received(:metric=).with(subject.namespaced_metric)
-          end
-
-          [[:register], [:do_close], [:multi_receive, [[]] ] ].each do |method, args|
-            context "strategy objects" do
-              before do
-                allow(out_inst).to receive(method)
-              end
-
-              it "should delegate #{method} to the strategy" do
-                subject.send(method, *args)
-                if args
-                  expect(out_inst).to have_received(method).with(*args)
-                else
-                  expect(out_inst).to have_received(method).with(no_args)
-                end
-              end
-            end
-
-            context "strategy output instances" do
-              before do
-                allow(out_inst).to receive(method)
-              end
-
-              it "should delegate #{method} to the strategy" do
-                subject.send(method, *args)
-                if args
-                  expect(out_inst).to have_received(method).with(*args)
-                else
-                  expect(out_inst).to have_received(method).with(no_args)
-                end
-              end
-            end
-          end
-        end
-      end
-    end
-  end
-end

data/spec/logstash/timestamp_spec.rb

@@ -1,45 +0,0 @@
-# encoding: utf-8
-
-require "spec_helper"
-
-describe LogStash::Timestamp do
-  context "constructors" do
-    # Via JRuby 9k time see logstash/issues/7463
-    # JRuby 9k now uses Java 8 Time with nanosecond precision but
-    # our Timestamp use Joda with millisecond precision
-    # expected: 2017-06-15 10:34:08.389999999 +0000
-    #      got: 2017-06-15 10:34:08.389000000 +0000
-    # we may need to use `be_within(0.000999999).of()` in other places too
-    it "should work" do
-      t = LogStash::Timestamp.new
-      expect(t.time.to_i).to be_within(2).of Time.now.to_i
-
-      t = LogStash::Timestamp.now
-      expect(t.time.to_i).to be_within(2).of Time.now.to_i
-
-      now = DateTime.now.to_time.utc
-      t = LogStash::Timestamp.new(now)
-      expect(t.time.to_f).to be_within(0.000999999).of(now.to_f)
-
-      t = LogStash::Timestamp.at(now.to_i)
-      expect(t.time.to_i).to eq(now.to_i)
-    end
-
-    it "should have consistent behaviour across == and .eql?" do
-      its_xmas = Time.utc(2015, 12, 25, 0, 0, 0)
-      expect(LogStash::Timestamp.new(its_xmas)).to eql(LogStash::Timestamp.new(its_xmas))
-      expect(LogStash::Timestamp.new(its_xmas)).to be ==(LogStash::Timestamp.new(its_xmas))
-    end
-
-    it "should raise exception on invalid format" do
-      expect{LogStash::Timestamp.new("foobar")}.to raise_error
-    end
-
-    it "compares to any type" do
-      t = LogStash::Timestamp.new
-      expect(t == '-').to be_falsey
-    end
-
-  end
-
-end