logstash-input-file 2.0.2 → 2.0.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 68859f099301705b718870b36fe7bf1f1347573f
-  data.tar.gz: 5916cfae7c42b57ddbd274b5ae3321a58e7d4eca
+  metadata.gz: 5377f5857709a60359d657a8007d6ab0b3f87f4d
+  data.tar.gz: 1af3cfea2b4135413c3cb347e61b7c0ff89985cc
 SHA512:
-  metadata.gz: 6bec077461e23f681a0bbd852edaa61015a7df065938e1f1b43842c1a3d662fd30eaa31bf4c42e43c276835d0144c4d65a247ac803dfa656878f367a502b4bcb
-  data.tar.gz: ca84de7eb86d11f34cc1540b5efe41a76bdbde907a637d66c96b97267d445843f86515f171ce2365085d9f86c7c850f14528c1226f103ed6b10c7bff97c739cc
+  metadata.gz: 2bd62efe503f31050170ebc8f17455f8c108bb0e0430a7ee1718250831292a90bf218bec728a418aa0cb3644cd587f50dadb35ce51ed6b5256e353bfe90d143b
+  data.tar.gz: 69ecdc00558c2b9da3a1d7e989e1a4936e0c24ab4a7f1c420bc64ba2dbd460f6751014821f71e25263b57dbd8e4598e637688233eee9233587ac73ceb58d776c

data/CHANGELOG.md

@@ -1,5 +1,15 @@
+## 2.0.3
+ - Implement Stream Identity mapping of codecs: distinct codecs will collect input per stream identity (filename)
+
+## 2.0.2
+ - Change LS core dependency version
+ - Add CI badge
+
+## 2.0.1
+ - Change LS core dependency version
+
 ## 2.0.0
- - Plugins were updated to follow the new shutdown semantic, this mainly allows Logstash to instruct input plugins to terminate gracefully, 
+ - Plugins were updated to follow the new shutdown semantic, this mainly allows Logstash to instruct input plugins to terminate gracefully,
    instead of using Thread.raise on the plugins' threads. Ref: https://github.com/elastic/logstash/pull/3895
  - Dependency on logstash-core update to 2.0
 

data/CONTRIBUTORS

@@ -17,6 +17,7 @@ Contributors:
 * Tejay Cardon (tejaycar)
 * elliot moore (em295)
 * yjpa7145
+* Guy Boertje (guyboertje)
 
 Note: If you've sent us patches, bug reports, or otherwise contributed to
 Logstash, and you aren't on the list above and want to be, please let us know

data/README.md

@@ -1,5 +1,8 @@
 # Logstash Plugin
 
+[![Build
+Status](http://build-eu-00.elastic.co/view/LS%20Plugins/view/LS%20Inputs/job/logstash-plugin-input-file-unit/badge/icon)](http://build-eu-00.elastic.co/view/LS%20Plugins/view/LS%20Inputs/job/logstash-plugin-input-file-unit/)
+
 This is a plugin for [Logstash](https://github.com/elastic/logstash).
 
 It is fully free and fully open source. The license is Apache 2.0, meaning you are pretty much free to use it however you want in whatever way.

data/lib/logstash/inputs/file.rb

@@ -1,6 +1,7 @@
 # encoding: utf-8
-require "logstash/inputs/base"
 require "logstash/namespace"
+require "logstash/inputs/base"
+require "logstash/codecs/identity_map_codec"
 
 require "pathname"
 require "socket" # for Socket.gethostname
@@ -181,27 +182,39 @@ class LogStash::Inputs::File < LogStash::Inputs::Base
     if @start_position == "beginning"
       @tail_config[:start_new_files_at] = :beginning
     end
+
+    @codec = LogStash::Codecs::IdentityMapCodec.new(@codec)
   end # def register
 
-  public
   def run(queue)
     @tail = FileWatch::Tail.new(@tail_config)
     @tail.logger = @logger
     @path.each { |path| @tail.tail(path) }
-
     @tail.subscribe do |path, line|
-      @logger.debug? && @logger.debug("Received line", :path => path, :text => line)
-      @codec.decode(line) do |event|
-        event["[@metadata][path]"] = path
-        event["host"] = @host if !event.include?("host")
-        event["path"] = path if !event.include?("path")
-        decorate(event)
-        queue << event
+      log_line_received(path, line)
+      @codec.decode(line, path) do |event|
+        # path is the identity
+        # Note: this block is cached in the
+        # identity_map_codec for use when
+        # buffered lines are flushed.
+        queue << add_path_meta(event, path)
       end
     end
   end # def run
 
-  public
+  def log_line_received(path, line)
+    return if !@logger.debug?
+    @logger.debug("Received line", :path => path, :text => line)
+  end
+
+  def add_path_meta(event, path)
+    event["[@metadata][path]"] = path
+    event["host"] = @host if !event.include?("host")
+    event["path"] = path if !event.include?("path")
+    decorate(event)
+    event
+  end
+
   def stop
     @tail.quit if @tail # _sincedb_write is called implicitly
   end

data/logstash-input-file.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-input-file'
-  s.version         = '2.0.2'
+  s.version         = '2.0.3'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = "Stream events from files."
   s.description     = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program"
@@ -25,6 +25,7 @@ Gem::Specification.new do |s|
   s.add_runtime_dependency 'logstash-codec-plain'
   s.add_runtime_dependency 'addressable'
   s.add_runtime_dependency 'filewatch', ['>= 0.6.5', '~> 0.6']
+  s.add_runtime_dependency 'logstash-codec-multiline', ['~> 2.0.3']
 
   s.add_development_dependency 'stud', ['~> 0.0.19']
   s.add_development_dependency 'logstash-devutils'

data/spec/inputs/file_spec.rb

@@ -5,9 +5,9 @@ require "tempfile"
 require "stud/temporary"
 require "logstash/inputs/file"
 
-describe LogStash::Inputs::File do
+FILE_DELIMITER = LogStash::Environment.windows? ? "\r\n" : "\n"
 
-  delimiter = (LogStash::Environment.windows? ? "\r\n" : "\n")
+describe LogStash::Inputs::File do
 
   it_behaves_like "an interruptible input plugin" do
     let(:config) do
@@ -28,7 +28,7 @@ describe LogStash::Inputs::File do
           type => "blah"
           path => "#{tmpfile_path}"
           sincedb_path => "#{sincedb_path}"
-          delimiter => "#{delimiter}"
+          delimiter => "#{FILE_DELIMITER}"
         }
       }
     CONFIG
@@ -82,7 +82,7 @@ describe LogStash::Inputs::File do
           path => "#{tmpfile_path}"
           start_position => "beginning"
           sincedb_path => "#{sincedb_path}"
-          delimiter => "#{delimiter}"
+          delimiter => "#{FILE_DELIMITER}"
         }
       }
     CONFIG
@@ -111,7 +111,7 @@ describe LogStash::Inputs::File do
           path => "#{tmpfile_path}"
           start_position => "beginning"
           sincedb_path => "#{sincedb_path}"
-          delimiter => "#{delimiter}"
+          delimiter => "#{FILE_DELIMITER}"
         }
       }
     CONFIG
@@ -154,7 +154,7 @@ describe LogStash::Inputs::File do
           path => "#{tmpfile_path}"
           start_position => "beginning"
           sincedb_path => "#{sincedb_path}"
-          delimiter => "#{delimiter}"
+          delimiter => "#{FILE_DELIMITER}"
           codec => "json"
         }
       }
@@ -189,4 +189,69 @@ describe LogStash::Inputs::File do
       expect { subject.register }.to raise_error(ArgumentError)
     end
   end
+
+  context "when wildcard path and a multiline codec is specified" do
+    let(:tmpdir_path)  { Stud::Temporary.directory }
+    let(:sincedb_path) { Stud::Temporary.pathname }
+    let(:conf) do
+      <<-CONFIG
+        input {
+          file {
+            type => "blah"
+            path => "#{tmpdir_path}/*.log"
+            start_position => "beginning"
+            sincedb_path => "#{sincedb_path}"
+            delimiter => "#{FILE_DELIMITER}"
+            codec => multiline { pattern => "^\s" what => previous }
+          }
+        }
+      CONFIG
+    end
+
+    let(:writer_proc) do
+      -> do
+        File.open("#{tmpdir_path}/a.log", "a") do |fd|
+          fd.puts("line1.1-of-a")
+          fd.puts("  line1.2-of-a")
+          fd.puts("  line1.3-of-a")
+          fd.puts("line2.1-of-a")
+        end
+
+        File.open("#{tmpdir_path}/z.log", "a") do |fd|
+          fd.puts("line1.1-of-z")
+          fd.puts("  line1.2-of-z")
+          fd.puts("  line1.3-of-z")
+          fd.puts("line2.1-of-z")
+        end
+      end
+    end
+
+    after do
+      FileUtils.rm_rf(tmpdir_path)
+    end
+
+    let(:event_count) { 2 }
+
+    it "collects separate multiple line events from each file" do
+      writer_proc.call
+
+      events = input(conf) do |pipeline, queue|
+        queue.size.times.collect { queue.pop }
+      end
+
+      expect(events.size).to eq(event_count)
+
+      e1_message = events[0]["message"]
+      e2_message = events[1]["message"]
+
+      # can't assume File A will be read first
+      if e1_message.start_with?('line1.1-of-z')
+        expect(e1_message).to eq("line1.1-of-z#{FILE_DELIMITER}  line1.2-of-z#{FILE_DELIMITER}  line1.3-of-z")
+        expect(e2_message).to eq("line1.1-of-a#{FILE_DELIMITER}  line1.2-of-a#{FILE_DELIMITER}  line1.3-of-a")
+      else
+        expect(e1_message).to eq("line1.1-of-a#{FILE_DELIMITER}  line1.2-of-a#{FILE_DELIMITER}  line1.3-of-a")
+        expect(e2_message).to eq("line1.1-of-z#{FILE_DELIMITER}  line1.2-of-z#{FILE_DELIMITER}  line1.3-of-z")
+      end
+    end
+  end
 end

metadata

@@ -1,17 +1,18 @@
 --- !ruby/object:Gem::Specification
 name: logstash-input-file
 version: !ruby/object:Gem::Version
-  version: 2.0.2
+  version: 2.0.3
 platform: ruby
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2015-10-14 00:00:00.000000000 Z
+date: 2015-11-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  requirement: !ruby/object:Gem::Requirement
+  name: logstash-core
+  version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
@@ -19,10 +20,7 @@ dependencies:
     - - <
       - !ruby/object:Gem::Version
         version: 3.0.0
-  name: logstash-core
-  prerelease: false
-  type: :runtime
-  version_requirements: !ruby/object:Gem::Requirement
+  requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
@@ -30,35 +28,46 @@ dependencies:
     - - <
       - !ruby/object:Gem::Version
         version: 3.0.0
+  prerelease: false
+  type: :runtime
 - !ruby/object:Gem::Dependency
+  name: logstash-codec-plain
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
         version: '0'
-  name: logstash-codec-plain
   prerelease: false
   type: :runtime
+- !ruby/object:Gem::Dependency
+  name: addressable
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
         version: '0'
-- !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
         version: '0'
-  name: addressable
   prerelease: false
   type: :runtime
+- !ruby/object:Gem::Dependency
+  name: filewatch
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
-        version: '0'
-- !ruby/object:Gem::Dependency
+        version: 0.6.5
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '0.6'
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
@@ -67,59 +76,64 @@ dependencies:
     - - ~>
       - !ruby/object:Gem::Version
         version: '0.6'
-  name: filewatch
   prerelease: false
   type: :runtime
+- !ruby/object:Gem::Dependency
+  name: logstash-codec-multiline
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
-      - !ruby/object:Gem::Version
-        version: 0.6.5
     - - ~>
       - !ruby/object:Gem::Version
-        version: '0.6'
-- !ruby/object:Gem::Dependency
+        version: 2.0.3
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 0.0.19
-  name: stud
+        version: 2.0.3
   prerelease: false
-  type: :development
+  type: :runtime
+- !ruby/object:Gem::Dependency
+  name: stud
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
         version: 0.0.19
-- !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ~>
       - !ruby/object:Gem::Version
-        version: '0'
-  name: logstash-devutils
+        version: 0.0.19
   prerelease: false
   type: :development
+- !ruby/object:Gem::Dependency
+  name: logstash-devutils
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
         version: '0'
-- !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
         version: '0'
-  name: logstash-codec-json
   prerelease: false
   type: :development
+- !ruby/object:Gem::Dependency
+  name: logstash-codec-json
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
         version: '0'
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  prerelease: false
+  type: :development
 description: This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program
 email: info@elastic.co
 executables: []
@@ -157,7 +171,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project:
-rubygems_version: 2.4.8
+rubygems_version: 2.4.5
 signing_key:
 specification_version: 4
 summary: Stream events from files.