logstash-input-tcp 0.1.3 → 0.1.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: d69d9bbbbb563037fa303e22890b2663d3d754fe
-  data.tar.gz: 91c386b38e725110193bf433070e07290b5935ba
+  metadata.gz: 1ccc75be3eea6fc13e2bf56bc7faef1dcce3b73d
+  data.tar.gz: a145775968ea16c80e66e175f252c9fd4cb4f46c
 SHA512:
-  metadata.gz: 073b544677fac9571b80b8d1f8ad858f350bbcbbfcf23d59d7e66b900df9603b8955acd81be04c8cdb23d65cb96b3e58f834a6629d05ecf76bdb3bb516f16da9
-  data.tar.gz: b628a21aef7d1e8a5ebb92bb3f3de594a590164854aa643ea45935b7e2f5acfa581c06a2c846dd598b35b578470807c57499e44ee9dbfbec1bcfba5523940dd9
+  metadata.gz: 8e0775708e8493ce73c417fb3aee887f5de16a8c6d1f2424ac806261407990a9bddfdb7cc5e0b6b31a91916fb2aa55c3bc0781d552c2082f53dd58e8cbf979b1
+  data.tar.gz: 59ae84706c2118eb28193b622c87a334a0eaaf5ee1c85c48cb6955392d68e0eb3a6540faa43d072e6f937c81e60449767bf8ef3a4de9edd298de2187ddf2a15d

data/lib/logstash/inputs/tcp.rb

@@ -10,7 +10,6 @@ require "logstash/util/socket_peer"
 # Can either accept connections from clients or connect to a server,
 # depending on `mode`.
 class LogStash::Inputs::Tcp < LogStash::Inputs::Base
-  class Interrupted < StandardError; end
   config_name "tcp"
 
   default :codec, "line"
@@ -54,6 +53,9 @@ class LogStash::Inputs::Tcp < LogStash::Inputs::Base
 
   def initialize(*args)
     super(*args)
+    @interrupted = false
+    @server_socket = nil
+    @client_socket = nil
   end # def initialize
 
   public
@@ -102,10 +104,10 @@ class LogStash::Inputs::Tcp < LogStash::Inputs::Base
 
   private
   def handle_socket(socket, client_address, output_queue, codec)
-    while true
+    while !@interrupted
       buf = nil
       # NOTE(petef): the timeout only hits after the line is read or socket dies
-      # TODO(sissel): Why do we have a timeout here? What's the point?
+      # TODO(sissel): Why do we have a Timeout here? What's the point?
       if @data_timeout == -1
         buf = read(socket)
       else
@@ -143,7 +145,8 @@ class LogStash::Inputs::Tcp < LogStash::Inputs::Base
       begin
         @logger.debug? && @logger.debug("Accepted connection", :client => s.peer, :server => "#{@host}:#{@port}")
         handle_socket(s, s.peeraddr[3], q, @codec.clone)
-      rescue Interrupted
+      rescue LogStash::ShutdownSignal
+        @interrupted = true
         s.close rescue nil
       ensure
         @client_threads_lock.synchronize{@client_threads.delete(Thread.current)}
@@ -158,7 +161,7 @@ class LogStash::Inputs::Tcp < LogStash::Inputs::Base
 
   private
   def read(socket)
-    return socket.sysread(16384)
+    socket.sysread(16384)
   end # def readline
 
   public
@@ -171,11 +174,10 @@ class LogStash::Inputs::Tcp < LogStash::Inputs::Base
   end # def run
 
   def run_server(output_queue)
-    @thread = Thread.current
     @client_threads = []
     @client_threads_lock = Mutex.new
 
-    while true
+    while !@interrupted
       begin
         socket = @server_socket.accept
         # start a new thread for each connection.
@@ -183,37 +185,28 @@ class LogStash::Inputs::Tcp < LogStash::Inputs::Base
       rescue OpenSSL::SSL::SSLError => ssle
         # NOTE(mrichar1): This doesn't return a useful error message for some reason
         @logger.error("SSL Error", :exception => ssle, :backtrace => ssle.backtrace)
-      rescue IOError, LogStash::ShutdownSignal
-        if @interrupted
-          @server_socket.close rescue nil
-
-          threads = @client_threads_lock.synchronize{@client_threads.dup}
-          threads.each do |thread|
-            thread.raise(LogStash::ShutdownSignal) if thread.alive?
-          end
-
-          # intended shutdown, get out of the loop
-          break
-        else
-          # it was a genuine IOError, propagate it up
-          raise
-        end
+      rescue IOError
+        raise unless @interrupted
       end
-    end # loop
+    end
   rescue LogStash::ShutdownSignal
-    # nothing to do
+    @interrupted = true
   ensure
     @server_socket.close rescue nil
+
+    threads = @client_threads_lock.synchronize{@client_threads.dup}
+    threads.each do |thread|
+      thread.raise(LogStash::ShutdownSignal) if thread.alive?
+    end
   end # def run_server
 
   def run_client(output_queue)
-    @thread = Thread.current
-    while true
-      client_socket = TCPSocket.new(@host, @port)
+    while !@interrupted
+      @client_socket = TCPSocket.new(@host, @port)
       if @ssl_enable
-        client_socket = OpenSSL::SSL::SSLSocket.new(client_socket, @ssl_context)
+        @client_socket = OpenSSL::SSL::SSLSocket.new(@client_socket, @ssl_context)
         begin
-          client_socket.connect
+          @client_socket.connect
         rescue OpenSSL::SSL::SSLError => ssle
           @logger.error("SSL Error", :exception => ssle, :backtrace => ssle.backtrace)
           # NOTE(mrichar1): Hack to prevent hammering peer
@@ -221,17 +214,23 @@ class LogStash::Inputs::Tcp < LogStash::Inputs::Base
           next
         end
       end
-      @logger.debug("Opened connection", :client => "#{client_socket.peer}")
-      handle_socket(client_socket, client_socket.peeraddr[3], output_queue, @codec.clone)
+      @logger.debug? && @logger.debug("Opened connection", :client => "#{@client_socket.peer}")
+      handle_socket(@client_socket, @client_socket.peeraddr[3], output_queue, @codec.clone)
     end # loop
   ensure
-    client_socket.close rescue nil
+    @client_socket.close rescue nil
   end # def run
 
   public
   def teardown
-    if server?
-      @interrupted = true
+    @interrupted = true
+    if @server_socket
+      @server_socket.close rescue nil
+      @server_socket = nil
+    end
+    if @client_socket
+      @client_socket.close rescue nil
+      @client_socket = nil
     end
   end # def teardown
 end # class LogStash::Inputs::Tcp

data/logstash-input-tcp.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-input-tcp'
-  s.version         = '0.1.3'
+  s.version         = '0.1.4'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = "Read events over a TCP socket."
   s.description     = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program"

data/spec/inputs/tcp_spec.rb

@@ -7,7 +7,7 @@ require "logstash/inputs/tcp"
 require 'stud/try'
 
 describe LogStash::Inputs::Tcp do
-  
+
 
   context "codec (PR #1372)" do
     it "switches from plain to line" do
@@ -16,6 +16,7 @@ describe LogStash::Inputs::Tcp do
       plugin = LogStash::Inputs::Tcp.new("codec" => LogStash::Codecs::Plain.new, "port" => 0)
       plugin.register
       insist { plugin.codec }.is_a?(LogStash::Codecs::Line)
+      plugin.teardown
     end
     it "switches from json to json_lines" do
       require "logstash/codecs/json"
@@ -23,13 +24,14 @@ describe LogStash::Inputs::Tcp do
       plugin = LogStash::Inputs::Tcp.new("codec" => LogStash::Codecs::JSON.new, "port" => 0)
       plugin.register
       insist { plugin.codec }.is_a?(LogStash::Codecs::JSONLines)
+      plugin.teardown
     end
   end
 
-  describe "read plain with unicode", :socket => true do
+  it "should read plain with unicode" do
     event_count = 10
     port = 5511
-    config <<-CONFIG
+    conf = <<-CONFIG
       input {
         tcp {
           port => #{port}
@@ -37,31 +39,30 @@ describe LogStash::Inputs::Tcp do
       }
     CONFIG
 
-    input do |pipeline, queue|
-      Thread.new { pipeline.run }
-      sleep 0.1 while !pipeline.ready?
 
+
+    events = input(conf) do |pipeline, queue|
       socket = Stud::try(5.times) { TCPSocket.new("127.0.0.1", port) }
       event_count.times do |i|
         # unicode smiley for testing unicode support!
         socket.puts("#{i} ☹")
+        socket.flush
       end
       socket.close
 
-      # wait till all events have been processed
-      Timeout.timeout(1) {sleep 0.1 while queue.size < event_count}
+      event_count.times.collect {queue.pop}
+    end
 
-      events = event_count.times.collect { queue.pop }
-      event_count.times do |i|
-        insist { events[i]["message"] } == "#{i} ☹"
-      end
-    end # input
+    insist { events.length } == event_count
+    event_count.times do |i|
+      insist { events[i]["message"] } == "#{i} ☹"
+    end
   end
 
-  describe "read events with plain codec and ISO-8859-1 charset" do
+  it "should read events with plain codec and ISO-8859-1 charset" do
     port = 5513
     charset = "ISO-8859-1"
-    config <<-CONFIG
+    conf = <<-CONFIG
       input {
         tcp {
           port => #{port}
@@ -70,32 +71,25 @@ describe LogStash::Inputs::Tcp do
       }
     CONFIG
 
-    input do |pipeline, queue|
-      Thread.new { pipeline.run }
-      sleep 0.1 while !pipeline.ready?
-
+    event = input(conf) do |pipeline, queue|
       socket = Stud::try(5.times) { TCPSocket.new("127.0.0.1", port) }
       text = "\xA3" # the £ symbol in ISO-8859-1 aka Latin-1
       text.force_encoding("ISO-8859-1")
       socket.puts(text)
       socket.close
 
-      # wait till all events have been processed
-      Timeout.timeout(1) {sleep 0.1 while queue.size < 1}
+      queue.pop
+    end
 
-      event = queue.pop
-      # Make sure the 0xA3 latin-1 code converts correctly to UTF-8.
-      pending("charset conv broken") do
-        insist { event["message"].size } == 1
-        insist { event["message"].bytesize } == 2
-        insist { event["message"] } == "£"
-      end
-    end # input
+    # Make sure the 0xA3 latin-1 code converts correctly to UTF-8.
+    insist { event["message"].size } == 1
+    insist { event["message"].bytesize } == 2
+    insist { event["message"] } == "£"
   end
 
-  describe "read events with json codec" do
+  it "should read events with json codec" do
     port = 5514
-    config <<-CONFIG
+    conf = <<-CONFIG
       input {
         tcp {
           port => #{port}
@@ -104,38 +98,33 @@ describe LogStash::Inputs::Tcp do
       }
     CONFIG
 
-    input do |pipeline, queue|
-      Thread.new { pipeline.run }
-      sleep 0.1 while !pipeline.ready?
-
-      data = {
-        "hello" => "world",
-        "foo" => [1,2,3],
-        "baz" => { "1" => "2" },
-        "host" => "example host"
-      }
+    data = {
+      "hello" => "world",
+      "foo" => [1,2,3],
+      "baz" => { "1" => "2" },
+      "host" => "example host"
+    }
 
+    event = input(conf) do |pipeline, queue|
       socket = Stud::try(5.times) { TCPSocket.new("127.0.0.1", port) }
       socket.puts(LogStash::Json.dump(data))
       socket.close
 
-      # wait till all events have been processed
-      Timeout.timeout(1) {sleep 0.1 while queue.size < 1}
+      queue.pop
+    end
 
-      event = queue.pop
-      insist { event["hello"] } == data["hello"]
-      insist { event["foo"].to_a } == data["foo"] # to_a to cast Java ArrayList produced by JrJackson
-      insist { event["baz"] } == data["baz"]
+    insist { event["hello"] } == data["hello"]
+    insist { event["foo"].to_a } == data["foo"] # to_a to cast Java ArrayList produced by JrJackson
+    insist { event["baz"] } == data["baz"]
 
-      # Make sure the tcp input, w/ json codec, uses the event's 'host' value,
-      # if present, instead of providing its own
-      insist { event["host"] } == data["host"]
-    end # input
+    # Make sure the tcp input, w/ json codec, uses the event's 'host' value,
+    # if present, instead of providing its own
+    insist { event["host"] } == data["host"]
   end
 
-  describe "read events with json codec (testing 'host' handling)" do
+  it "should read events with json codec (testing 'host' handling)" do
     port = 5514
-    config <<-CONFIG
+    conf = <<-CONFIG
       input {
         tcp {
           port => #{port}
@@ -144,30 +133,25 @@ describe LogStash::Inputs::Tcp do
       }
     CONFIG
 
-    input do |pipeline, queue|
-      Thread.new { pipeline.run }
-      sleep 0.1 while !pipeline.ready?
-
-      data = {
-        "hello" => "world"
-      }
+    data = {
+      "hello" => "world"
+    }
 
+    event = input(conf) do |pipeline, queue|
       socket = Stud::try(5.times) { TCPSocket.new("127.0.0.1", port) }
       socket.puts(LogStash::Json.dump(data))
       socket.close
 
-      # wait till all events have been processed
-      Timeout.timeout(1) {sleep 0.1 while queue.size < 1}
+      queue.pop
+    end
 
-      event = queue.pop
-      insist { event["hello"] } == data["hello"]
-      insist { event }.include?("host")
-    end # input
+    insist { event["hello"] } == data["hello"]
+    insist { event }.include?("host")
   end
 
-  describe "read events with json_lines codec" do
+  it "should read events with json_lines codec" do
     port = 5515
-    config <<-CONFIG
+    conf = <<-CONFIG
       input {
         tcp {
           port => #{port}
@@ -176,38 +160,36 @@ describe LogStash::Inputs::Tcp do
       }
     CONFIG
 
-    input do |pipeline, queue|
-      Thread.new { pipeline.run }
-      sleep 0.1 while !pipeline.ready?
-
-      data = {
-        "hello" => "world",
-        "foo" => [1,2,3],
-        "baz" => { "1" => "2" },
-        "idx" => 0
-      }
+    data = {
+      "hello" => "world",
+      "foo" => [1,2,3],
+      "baz" => { "1" => "2" },
+      "idx" => 0
+    }
 
+    events = input(conf) do |pipeline, queue|
       socket = Stud::try(5.times) { TCPSocket.new("127.0.0.1", port) }
       (1..5).each do |idx|
         data["idx"] = idx
         socket.puts(LogStash::Json.dump(data) + "\n")
-      end # do
+      end
       socket.close
 
-      (1..5).each do |idx|
-        event = queue.pop
-        insist { event["hello"] } == data["hello"]
-        insist { event["foo"].to_a } == data["foo"] # to_a to cast Java ArrayList produced by JrJackson
-        insist { event["baz"] } == data["baz"]
-        insist { event["idx"] } == idx
-      end # do
-    end # input
+      (1..5).map{queue.pop}
+    end
+
+    events.each_with_index do |event, idx|
+      insist { event["hello"] } == data["hello"]
+      insist { event["foo"].to_a } == data["foo"] # to_a to cast Java ArrayList produced by JrJackson
+      insist { event["baz"] } == data["baz"]
+      insist { event["idx"] } == idx + 1
+    end # do
   end # describe
 
-  describe "one message per connection" do
+  it "should one message per connection" do
     event_count = 10
     port = 5516
-    config <<-CONFIG
+    conf = <<-CONFIG
       input {
         tcp {
           port => #{port}
@@ -215,33 +197,27 @@ describe LogStash::Inputs::Tcp do
       }
     CONFIG
 
-    input do |pipeline, queue|
-      Thread.new { pipeline.run }
-      sleep 0.1 while !pipeline.ready?
-
+    events = input(conf) do |pipeline, queue|
       event_count.times do |i|
         socket = Stud::try(5.times) { TCPSocket.new("127.0.0.1", port) }
         socket.puts("#{i}")
         socket.flush
         socket.close
       end
-      
-      # wait till all events have been processed
-      Timeout.timeout(1) {sleep 0.1 while queue.size < event_count}
 
       # since each message is sent on its own tcp connection & thread, exact receiving order cannot be garanteed
-      events = event_count.times.collect{queue.pop}.sort_by{|event| event["message"]}
+      event_count.times.collect{queue.pop}.sort_by{|event| event["message"]}
+    end
 
-      event_count.times do |i|
-        insist { events[i]["message"] } == "#{i}"
-      end
-    end # input
+    event_count.times do |i|
+      insist { events[i]["message"] } == "#{i}"
+    end
   end
 
-  describe "connection threads are cleaned up when connection is closed" do
+  it "should connection threads are cleaned up when connection is closed" do
     event_count = 10
     port = 5517
-    config <<-CONFIG
+    conf = <<-CONFIG
       input {
         tcp {
           port => #{port}
@@ -249,10 +225,7 @@ describe LogStash::Inputs::Tcp do
       }
     CONFIG
 
-    input do |pipeline, queue|
-      Thread.new { pipeline.run }
-      sleep 0.1 while !pipeline.ready?
-
+    input(conf) do |pipeline, queue|
       inputs = pipeline.instance_variable_get("@inputs")
       insist { inputs.size } == 1
 
@@ -263,18 +236,13 @@ describe LogStash::Inputs::Tcp do
         socket
       end
 
-      # wait till all events have been processed
-      Timeout.timeout(1) {sleep 0.1 while queue.size < event_count}
-
-      # we should have "event_count" pending threads since sockets were not closed yet
       client_threads = inputs[0].instance_variable_get("@client_threads")
-      insist { client_threads.size } == event_count
 
       # close all sockets and make sure there is not more pending threads
       sockets.each{|socket| socket.close}
+
       Timeout.timeout(1) {sleep 0.1 while client_threads.size > 0}
       insist { client_threads.size } == 0 # this check is actually useless per previous line
-
-    end # input
+    end
   end
 end

metadata

@@ -1,17 +1,18 @@
 --- !ruby/object:Gem::Specification
 name: logstash-input-tcp
 version: !ruby/object:Gem::Version
-  version: 0.1.3
+  version: 0.1.4
 platform: ruby
 authors:
 - Elasticsearch
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2015-02-26 00:00:00.000000000 Z
+date: 2015-04-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  requirement: !ruby/object:Gem::Requirement
+  name: logstash-core
+  version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
@@ -19,10 +20,7 @@ dependencies:
     - - <
       - !ruby/object:Gem::Version
         version: 2.0.0
-  name: logstash-core
-  prerelease: false
-  type: :runtime
-  version_requirements: !ruby/object:Gem::Requirement
+  requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
@@ -30,76 +28,78 @@ dependencies:
     - - <
       - !ruby/object:Gem::Version
         version: 2.0.0
+  prerelease: false
+  type: :runtime
 - !ruby/object:Gem::Dependency
+  name: logstash-codec-plain
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
         version: '0'
-  name: logstash-codec-plain
   prerelease: false
   type: :runtime
+- !ruby/object:Gem::Dependency
+  name: logstash-codec-line
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
         version: '0'
-- !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
         version: '0'
-  name: logstash-codec-line
   prerelease: false
   type: :runtime
+- !ruby/object:Gem::Dependency
+  name: logstash-codec-json
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
         version: '0'
-- !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
         version: '0'
-  name: logstash-codec-json
   prerelease: false
   type: :runtime
+- !ruby/object:Gem::Dependency
+  name: logstash-codec-json_lines
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
         version: '0'
-- !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
         version: '0'
-  name: logstash-codec-json_lines
   prerelease: false
   type: :runtime
+- !ruby/object:Gem::Dependency
+  name: logstash-devutils
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
         version: '0'
-- !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
         version: '0'
-  name: logstash-devutils
   prerelease: false
   type: :development
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - '>='
-      - !ruby/object:Gem::Version
-        version: '0'
 description: This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program
 email: info@elasticsearch.com
 executables: []
@@ -137,7 +137,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project:
-rubygems_version: 2.4.5
+rubygems_version: 2.1.9
 signing_key:
 specification_version: 4
 summary: Read events over a TCP socket.