logstash-codec-multiline 2.0.2 → 2.0.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: f2a90b45c2f55e7db24a9f2af3905b82d18bba6c
-  data.tar.gz: 27cdceb50b94688c6902d10593db9f2ff4db33d3
+  metadata.gz: b89e4a5a454c8bc979aca1a959c06ad2d65e29bc
+  data.tar.gz: 3eff2f29480107e6212eec95c096c18b3f82b55e
 SHA512:
-  metadata.gz: d040c49a3fba07cd543e1eedc4a608c90c03bd5106e7c431fa85425638d3ce05d5c43a4c00289af0a531f819ca2034a5e57d6bd127df75af556562b22985fefa
-  data.tar.gz: 5ff667aa293cc3bdbfeba833ede8127041f7eb81b123a9a6a4afd641f9092d8452e570d6bd8b517fcbb88873a0d768d6e8852b0a9e2330ca2bcf867ccbf45cfc
+  metadata.gz: d7eb64acd24053b8ce18bfe111a5ab2917f5a41a3e979879ee1313bc64ffcf744ed85d11ade9843167bfe86579a34b8015d62b5fa0ee85d1356a1ac3360d246e
+  data.tar.gz: 36580542f95f5b66d042f8a2e6ed69064b49a1e2c1e2400f0aa49bfeb8f54ec510b99646fc1758d3a9887fb41862d1fd7ad44c30cbf3be721bf73323221953af

data/CHANGELOG.md

@@ -1,5 +1,8 @@
+## 2.0.3
+ - Add pseudo codec IdentityMapCodec.  Support class for identity based multiline processing.
+
 ## 2.0.0
- - Plugins were updated to follow the new shutdown semantic, this mainly allows Logstash to instruct input plugins to terminate gracefully, 
+ - Plugins were updated to follow the new shutdown semantic, this mainly allows Logstash to instruct input plugins to terminate gracefully,
    instead of using Thread.raise on the plugins' threads. Ref: https://github.com/elastic/logstash/pull/3895
  - Dependency on logstash-core update to 2.0
 

data/README.md

@@ -1,5 +1,8 @@
 # Logstash Plugin
 
+[![Build
+Status](http://build-eu-00.elastic.co/view/LS%20Plugins/view/LS%20Codecs/job/logstash-plugin-codec-multiline-unit/badge/icon)](http://build-eu-00.elastic.co/view/LS%20Plugins/view/LS%20Codecs/job/logstash-plugin-codec-multiline-unit/)
+
 This is a plugin for [Logstash](https://github.com/elastic/logstash).
 
 It is fully free and fully open source. The license is Apache 2.0, meaning you are pretty much free to use it however you want in whatever way.

data/lib/logstash/codecs/identity_map_codec.rb

@@ -0,0 +1,251 @@
+# encoding: utf-8
+require "logstash/namespace"
+require "thread_safe"
+
+# This class is a Codec duck type
+# Using Composition, it maps from a stream identity to
+# a cloned codec instance via the same API as a Codec
+# it implements the codec public API
+
+module LogStash module Codecs class IdentityMapCodec
+  # subclass of Exception, LS has more than limit (20000) active streams
+  class IdentityMapUpperLimitException < Exception; end
+
+  module EightyPercentWarning
+    extend self
+    def visit(imc)
+      current_size, limit = imc.current_size_and_limit
+      return if current_size < (limit * 0.8)
+      imc.logger.warn("IdentityMapCodec has reached 80% capacity",
+        :current_size => current_size, :upper_limit => limit)
+    end
+  end
+
+  module UpperLimitReached
+    extend self
+    def visit(imc)
+      current_size, limit = imc.current_size_and_limit
+      return if current_size < limit
+      # we hit the limit
+      # try to clean out stale streams
+      current_size, limit = imc.map_cleanup
+      return if current_size < limit
+      # we are still at the limit and all streams are in use
+      imc.logger.error("IdentityMapCodec has reached 100% capacity",
+          :current_size => current_size, :upper_limit => limit)
+      raise IdentityMapUpperLimitException.new
+    end
+  end
+
+  class MapCleaner
+    def initialize(imc, interval)
+      @imc, @interval = imc, interval
+      @running = false
+    end
+
+    def start
+      return self if running?
+      @running = true
+      @thread = Thread.new(@imc) do |imc|
+        loop do
+          sleep @interval
+          break if !@running
+          imc.map_cleanup
+        end
+      end
+      self
+    end
+
+    def running?
+      @running
+    end
+
+    def stop
+      return if !running?
+      @running = false
+      @thread.wakeup
+    end
+  end
+
+  # A composite class to hold both the codec and the eviction_timeout
+  # instances of this Value Object are stored in the mapping hash
+  class CodecValue
+    attr_reader :codec
+    attr_accessor :timeout
+
+    def initialize(codec)
+      @codec = codec
+    end
+  end
+
+  #maximum size of the mapping hash
+  MAX_IDENTITIES = 20_000
+
+  # time after which a stream is
+  # considered stale
+  # each time a stream is accessed
+  # it is given a new timeout
+  EVICT_TIMEOUT = 60 * 60 * 1 # 1 hour
+
+  # time that the cleaner thread sleeps for
+  # before it tries to clean out stale mappings
+  CLEANER_INTERVAL = 60 * 5 # 5 minutes
+
+  attr_reader :identity_map
+  attr_accessor :base_codec, :cleaner
+
+  def initialize(codec)
+    @base_codec = codec
+    @base_codecs = [codec]
+    @identity_map = ThreadSafe::Hash.new &method(:codec_builder)
+    @max_identities = MAX_IDENTITIES
+    @evict_timeout = EVICT_TIMEOUT
+    @cleaner = MapCleaner.new(self, CLEANER_INTERVAL)
+    @decode_block = lambda {|*| }
+  end
+
+  # ==============================================
+  # Constructional/builder methods
+  # chain this method off of new
+  #
+  # used to add a non-default maximum identities
+  def max_identities(max)
+    @max_identities = max.to_i
+    self
+  end
+
+  # used to add a non-default evict timeout
+  def evict_timeout(timeout)
+    @evict_timeout = timeout.to_i
+    self
+  end
+
+  # used to add  a non-default cleaner interval
+  def cleaner_interval(interval)
+    @cleaner.stop
+    @cleaner = MapCleaner.new(self, interval.to_i)
+    self
+  end
+  # end Constructional/builder methods
+  # ==============================================
+
+  # ==============================================
+  # Codec API
+  def decode(data, identity = nil, &block)
+    @decode_block = block if @decode_block != block
+    stream_codec(identity).decode(data, &block)
+  end
+
+  alias_method :<<, :decode
+
+  def encode(event, identity = nil)
+    stream_codec(identity).encode(event)
+  end
+
+  # this method will not be called from
+  # the input or the pipeline unless
+  # we implement codec flush on shutdown
+  # problematic, because we may not have
+  # received all the multiline parts yet.
+  # but if we don't flush we will lose data
+  def flush(&block)
+    all_codecs.each do |codec|
+      #let ruby do its default args thing
+      block.nil? ? codec.flush : codec.flush(&block)
+    end
+  end
+
+  def close()
+    cleaner.stop
+    all_codecs.each(&:close)
+  end
+  # end Codec API
+  # ==============================================
+
+  def all_codecs
+    no_streams? ? @base_codecs : identity_map.values.map(&:codec)
+  end
+
+  def max_limit
+    @max_identities
+  end
+
+  def identity_count
+    identity_map.size
+  end
+
+  # support cleaning of stale stream/codecs
+  # a stream is considered stale if it has not
+  # been accessed in the last @evict_timeout
+  # period (default 1 hour)
+  def map_cleanup
+    cut_off = Time.now.to_i
+    # delete_if is atomic
+    # contents should not mutate during this call
+    identity_map.delete_if do |identity, compo|
+      if (flag = compo.timeout <= cut_off)
+        compo.codec.flush(&@decode_block)
+      end
+      flag
+    end
+    current_size_and_limit
+  end
+
+  def current_size_and_limit
+    [identity_count, max_limit]
+  end
+
+  def logger
+    # we 'borrow' the codec's logger as we don't have our own
+    @base_codec.logger
+  end
+
+  def codec_without_usage_update(identity)
+    find_codec_value(identity).codec
+  end
+
+  def eviction_timestamp_for(identity)
+    find_codec_value(identity).timeout
+  end
+
+  private
+
+  def stream_codec(identity)
+    return base_codec if identity.nil?
+    record_codec_usage(identity) # returns codec
+  end
+
+  def find_codec_value(identity)
+    identity_map[identity]
+  end
+
+  # for nil stream this method is not called
+  def record_codec_usage(identity)
+    check_map_limits
+    # only start the cleaner if streams are in use
+    # continuous calls to start are OK
+    cleaner.start
+    compo = find_codec_value(identity)
+    compo.timeout = eviction_timestamp
+    compo.codec
+  end
+
+  def eviction_timestamp
+    Time.now.to_i + @evict_timeout
+  end
+
+  def check_map_limits
+    UpperLimitReached.visit(self)
+    EightyPercentWarning.visit(self)
+  end
+
+  def codec_builder(hash, k)
+    codec = hash.empty? ? @base_codec : @base_codec.clone
+    compo = CodecValue.new(codec)
+    hash.store(k, compo)
+  end
+
+  def no_streams?
+    identity_map.empty?
+  end
+end end end

data/lib/logstash/codecs/multiline.rb

@@ -181,7 +181,7 @@ class LogStash::Codecs::Multiline < LogStash::Codecs::Base
   end
 
   def flush(&block)
-    if @buffer.any? 
+    if @buffer.any?
       yield merge_events
       reset_buffer
     end
@@ -211,7 +211,7 @@ class LogStash::Codecs::Multiline < LogStash::Codecs::Base
   end
 
   def over_maximun_lines?
-    @buffer.size > @max_lines 
+    @buffer.size > @max_lines
   end
 
   def over_maximun_bytes?
@@ -227,4 +227,4 @@ class LogStash::Codecs::Multiline < LogStash::Codecs::Base
     @on_event.call(event, event)
   end # def encode
 
-end # class LogStash::Codecs::Plain
+end # class LogStash::Codecs::Multiline

data/logstash-codec-multiline.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-codec-multiline'
-  s.version         = '2.0.2'
+  s.version         = '2.0.3'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = "The multiline codec will collapse multiline messages and merge them into a single event."
   s.description     = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program"

data/spec/codecs/identity_map_codec_spec.rb

@@ -0,0 +1,208 @@
+# encoding: utf-8
+require "logstash/devutils/rspec/spec_helper"
+require "logstash/codecs/identity_map_codec"
+
+class LogTracer
+  def initialize() @tracer = []; end
+  def warn(*args) @tracer.push [:warn, args]; end
+  def error(*args) @tracer.push [:error, args]; end
+
+  def trace_for(symbol)
+    params = @tracer.assoc(symbol)
+    params.nil? ? false : params.last
+  end
+end
+
+class IdentityMapCodecTracer
+  def initialize() @tracer = []; end
+  def clone() self.class.new; end
+  def decode(data) @tracer.push [:decode, data]; end
+  def encode(event) @tracer.push [:encode, event]; end
+  def flush(&block) @tracer.push [:flush, true]; end
+  def close() @tracer.push [:close, true]; end
+  def logger() @logger ||= LogTracer.new; end
+
+  def trace_for(symbol)
+    params = @tracer.assoc(symbol)
+    params.nil? ? false : params.last
+  end
+end
+
+describe LogStash::Codecs::IdentityMapCodec do
+  let(:codec)   { IdentityMapCodecTracer.new }
+  let(:logger)  { codec.logger }
+  let(:demuxer) { described_class.new(codec) }
+  let(:stream1) { "stream-a" }
+  let(:codec1)  { demuxer.codec_without_usage_update(stream1) }
+  let(:arg1)    { "data-a" }
+
+  after do
+    codec.close
+  end
+
+  describe "operating without stream identity" do
+    let(:stream1) { nil }
+
+    it "transparently refers to the original codec" do
+      expect(codec).to eql(codec1)
+    end
+  end
+
+  describe "operating with stream identity" do
+
+    before { demuxer.decode(arg1, stream1) }
+
+    it "the first identity refers to the original codec" do
+      expect(codec).to eql(codec1)
+    end
+  end
+
+  describe "#decode" do
+    context "when no identity is used" do
+      let(:stream1) { nil }
+
+      it "calls the method on the original codec" do
+        demuxer.decode(arg1, stream1)
+
+        expect(codec.trace_for(:decode)).to eq(arg1)
+      end
+    end
+
+    context "when multiple identities are used" do
+      let(:stream2) { "stream-b" }
+      let(:codec2) { demuxer.codec_without_usage_update(stream2) }
+      let(:arg2)   { "data-b" }
+
+      it "calls the method on the appropriate codec" do
+        demuxer.decode(arg1, stream1)
+        demuxer.decode(arg2, stream2)
+
+        expect(codec1.trace_for(:decode)).to eq(arg1)
+        expect(codec2.trace_for(:decode)).to eq(arg2)
+      end
+    end
+  end
+
+  describe "#encode" do
+    context "when no identity is used" do
+      let(:stream1) { nil }
+      let(:arg1) { LogStash::Event.new({"type" => "file"}) }
+
+      it "calls the method on the original codec" do
+        demuxer.encode(arg1, stream1)
+
+        expect(codec.trace_for(:encode)).to eq(arg1)
+      end
+    end
+
+    context "when multiple identities are used" do
+      let(:stream2) { "stream-b" }
+      let(:codec2) { demuxer.codec_without_usage_update(stream2) }
+      let(:arg2)   { LogStash::Event.new({"type" => "file"}) }
+
+      it "calls the method on the appropriate codec" do
+        demuxer.encode(arg1, stream1)
+        demuxer.encode(arg2, stream2)
+
+        expect(codec1.trace_for(:encode)).to eq(arg1)
+        expect(codec2.trace_for(:encode)).to eq(arg2)
+      end
+    end
+  end
+
+  describe "#close" do
+    context "when no identity is used" do
+      before do
+        demuxer.decode(arg1)
+      end
+
+      it "calls the method on the original codec" do
+        demuxer.close
+        expect(codec.trace_for(:close)).to be_truthy
+      end
+    end
+
+    context "when multiple identities are used" do
+      let(:stream2) { "stream-b" }
+      let(:codec2) { demuxer.codec_without_usage_update(stream2) }
+      let(:arg2)   { LogStash::Event.new({"type" => "file"}) }
+
+      before do
+        demuxer.decode(arg1, stream1)
+        demuxer.decode(arg2, stream2)
+      end
+
+      it "calls the method on all codecs" do
+        demuxer.close
+
+        expect(codec1.trace_for(:close)).to be_truthy
+        expect(codec2.trace_for(:close)).to be_truthy
+      end
+    end
+  end
+
+  describe "over capacity protection" do
+    let(:demuxer) { described_class.new(codec).max_identities(limit) }
+
+    context "when capacity at 80% or higher" do
+      let(:limit) { 10 }
+
+      it "a warning is logged" do
+        limit.pred.times do |i|
+          demuxer.decode(Object.new, "stream#{i}")
+        end
+
+        expect(logger.trace_for(:warn).first).to match %r|has reached 80% capacity|
+      end
+    end
+
+    context "when capacity is exceeded" do
+      let(:limit) { 2 }
+      let(:error_class) { LogStash::Codecs::IdentityMapCodec::IdentityMapUpperLimitException }
+
+      it "an exception is raised" do
+        limit.times do |i|
+          demuxer.decode(Object.new, "stream#{i}")
+        end
+        expect { demuxer.decode(Object.new, "stream4") }.to raise_error(error_class)
+      end
+
+      context "initially but some streams are idle and can be evicted" do
+        let(:demuxer) { described_class.new(codec).max_identities(limit).evict_timeout(1) }
+
+        it "an exception is NOT raised" do
+          demuxer.decode(Object.new, "stream1")
+          sleep(1.2)
+          demuxer.decode(Object.new, "stream2")
+          expect(demuxer.identity_count).to eq(limit)
+          expect { demuxer.decode(Object.new, "stream4") }.not_to raise_error
+        end
+      end
+    end
+  end
+
+  describe "usage tracking" do
+    let(:demuxer) { described_class.new(codec).evict_timeout(10) }
+    context "when an operation is performed by identity" do
+      it "the new eviction time for that identity is recorded" do
+        demuxer.decode(Object.new, "stream1")
+        current_eviction = demuxer.eviction_timestamp_for("stream1")
+        sleep(2)
+        demuxer.decode(Object.new, "stream1")
+        expect(demuxer.eviction_timestamp_for("stream1")).to be >= current_eviction + 2
+      end
+    end
+  end
+
+  describe "codec eviction" do
+    let(:demuxer) { described_class.new(codec).evict_timeout(1).cleaner_interval(1) }
+    context "when an identity has become stale" do
+      it "the cleaner evicts the codec and flushes it first" do
+        demuxer.decode(Object.new, "stream1")
+        sleep(2.1)
+        expect(codec.trace_for(:flush)).to be_truthy
+        expect(demuxer.identity_map.keys).not_to include("stream1")
+      end
+    end
+  end
+end

metadata

@@ -1,17 +1,18 @@
 --- !ruby/object:Gem::Specification
 name: logstash-codec-multiline
 version: !ruby/object:Gem::Version
-  version: 2.0.2
+  version: 2.0.3
 platform: ruby
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2015-10-14 00:00:00.000000000 Z
+date: 2015-11-11 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  requirement: !ruby/object:Gem::Requirement
+  name: logstash-core
+  version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
@@ -19,10 +20,7 @@ dependencies:
     - - <
       - !ruby/object:Gem::Version
         version: 3.0.0
-  name: logstash-core
-  prerelease: false
-  type: :runtime
-  version_requirements: !ruby/object:Gem::Requirement
+  requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
@@ -30,64 +28,68 @@ dependencies:
     - - <
       - !ruby/object:Gem::Version
         version: 3.0.0
+  prerelease: false
+  type: :runtime
 - !ruby/object:Gem::Dependency
+  name: logstash-patterns-core
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
         version: '0'
-  name: logstash-patterns-core
   prerelease: false
   type: :runtime
+- !ruby/object:Gem::Dependency
+  name: jls-grok
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ~>
       - !ruby/object:Gem::Version
-        version: '0'
-- !ruby/object:Gem::Dependency
+        version: 0.11.1
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
         version: 0.11.1
-  name: jls-grok
   prerelease: false
   type: :runtime
+- !ruby/object:Gem::Dependency
+  name: logstash-devutils
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - '>='
       - !ruby/object:Gem::Version
-        version: 0.11.1
-- !ruby/object:Gem::Dependency
+        version: '0'
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
         version: '0'
-  name: logstash-devutils
   prerelease: false
   type: :development
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - '>='
-      - !ruby/object:Gem::Version
-        version: '0'
 description: This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program
 email: info@elastic.co
 executables: []
 extensions: []
 extra_rdoc_files: []
 files:
+- lib/logstash/codecs/identity_map_codec.rb
+- lib/logstash/codecs/multiline.rb
+- spec/codecs/multiline_spec.rb
+- spec/codecs/identity_map_codec_spec.rb
+- spec/supports/helpers.rb
+- logstash-codec-multiline.gemspec
+- README.md
 - CHANGELOG.md
 - CONTRIBUTORS
 - Gemfile
 - LICENSE
 - NOTICE.TXT
-- README.md
-- lib/logstash/codecs/multiline.rb
-- logstash-codec-multiline.gemspec
-- spec/codecs/multiline_spec.rb
-- spec/supports/helpers.rb
 homepage: http://www.elastic.co/guide/en/logstash/current/index.html
 licenses:
 - Apache License (2.0)
@@ -110,10 +112,11 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project:
-rubygems_version: 2.4.8
+rubygems_version: 2.1.9
 signing_key:
 specification_version: 4
 summary: The multiline codec will collapse multiline messages and merge them into a single event.
 test_files:
 - spec/codecs/multiline_spec.rb
+- spec/codecs/identity_map_codec_spec.rb
 - spec/supports/helpers.rb