logstash-output-logmatic 0.1.1 → 0.1.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 1f4c336a1aa8c1c96d35e9dfb0527bb3be5bbb1a
-  data.tar.gz: 6c7963ab72043cda85de06d3ea34d8b2fa22b44d
+  metadata.gz: a61c28d9d73583df0f16c50fd0f02429e35270e9
+  data.tar.gz: b5d6bc8647be4c4b25eabb6ce073d6a5b6c7559b
 SHA512:
-  metadata.gz: 388b0d88dfd686fc2c1f98887f863e31cb5e52831f5c45962f8e3db043abbd38ee660da1e0f963392a659336f203fea39defb62a88ab8cbb0098d6313b8c56e9
-  data.tar.gz: b0168749fd2c202341e3c313da94da9e73420c4e55bef81a3d6d7375dc1ea64c603d39d6b9b09eab575fc69cdaaacfd253b8984ffc6b9fa5fa36c518ded7aa83
+  metadata.gz: a6d1ddd1a910ff2cd32e4e77c65bb80b9e83e3e60fed3d625683acb31f7160344a5bce52e6a9e2eb427e53359283d12f39f33ae2b226d41876c7c43b14a2a0ba
+  data.tar.gz: 3c0ec9702a3f2a0819564d1a13d33169e0403bf65e99f23006f5879a11d61ca825e9a576613648713e5a88243d1bee96c2f96adfa63c87777c8e019209b72080

data/README.md

@@ -1,6 +1,6 @@
 # Logstash Logmatic.io's Output Plugin
 
-TThis project is an output plugin to send logs to Logmatic.io from Logstash >v1.5.0.
+This project is an output plugin to send logs to Logmatic.io from Logstash >v1.5.0.
 
 ## How to install it?
 
@@ -27,12 +27,6 @@ Then you just need to configure the output as follow:
 output {
     logmatic {
         key => "<your_api_key>"
-        # In order to limit the amount of resources the output uses buffering
-        # You can configure its behavior with:
-        # The number of elements queued before flushing
-        # queue_size => 10
-        # The timeframe in seconds for buffering before flushing
-        # timeframe => 10
     }
 }
 

data/lib/logstash/outputs/logmatic.rb

@@ -1,65 +1,150 @@
 # encoding: utf-8
 require "logstash/outputs/base"
 require "logstash/namespace"
-require "stud/buffer"
+require "thread"
 
-# Ship log from logstash straight to Logmatic
+# Write events over a TCP socket.
 #
-# To use this you will need a valid Logmatic API Key
-class LogStash::Outputs::LogmaticBatch < LogStash::Outputs::Base
-  include Stud::Buffer
+# Each event json is separated by a newline.
+#
+# Can either accept connections from clients or connect to a server,
+# depending on `mode`.
+class LogStash::Outputs::LogmaticTcp < LogStash::Outputs::Base
 
   config_name "logmatic"
-  milestone 2
 
   default :codec, "json"
 
+  # When mode is `server`, the address to listen on.
+  # When mode is `client`, the address to connect to.
+  config :host, :validate => :string, :default => "api.logmatic.io"
+
+  # When mode is `server`, the port to listen on.
+  # When mode is `client`, the port to connect to.
+  config :port, :validate => :number, :default => 10514
+
   # The Logmatic api key
   # You can find it in the 'Account' section in the Logmatic interface.
   config :key, :validate => :string, :required => true
 
-  # How many events to queue before flushing to Logmatic
-  # prior to schedule set in @timeframe
-  config :queue_size, :validate => :number, :default => 10
+  # When connect failed,retry interval in sec.
+  config :reconnect_interval, :validate => :number, :default => 10
 
-  # How often (in seconds) to flush queued events to Logmatic
-  config :timeframe, :validate => :number, :default => 10
+  # Mode to operate in. `server` listens for client connections,
+  # `client` connects to a server.
+  config :mode, :validate => ["server", "client"], :default => "client"
+
+  # The format to use when writing events to the file. This value
+  # supports any string and can include `%{name}` and other dynamic
+  # strings.
+  #
+  # If this setting is omitted, the full json representation of the
+  # event will be written as a single line.
+  config :message_format, :validate => :string, :deprecated => true
+
+  class Client
+    public
+    def initialize(socket, logger)
+      @socket = socket
+      @logger = logger
+      @queue  = Queue.new
+    end
+
+    public
+    def run
+      loop do
+        begin
+          @socket.write(@queue.pop)
+        rescue => e
+          @logger.warn("tcp output exception", :socket => @socket,
+                       :exception => e)
+          break
+        end
+      end
+    end # def run
+
+    public
+    def write(msg)
+      @queue.push(msg)
+    end # def write
+  end # class Client
 
   public
   def register
-    require "net/https"
-    require "uri"
-    @url = "https://api.logmatic.io/v1/input/#{@key}"
-    @uri = URI.parse(@url)
-    @client = Net::HTTP.new(@uri.host, @uri.port)
-    @client.use_ssl = true
-    @client.verify_mode = OpenSSL::SSL::VERIFY_NONE
-    @logger.debug("Logmatic URL", :url => @url)
-
-    buffer_initialize(
-      :max_items => @queue_size,
-      :max_interval => @timeframe,
-      :logger => @logger
-    )
-  end
+    require "socket"
+    require "stud/try"
+    if server?
+      workers_not_supported
 
-  public
-  def receive(event)
-    return unless output?(event)
-    buffer_receive(event.to_json)
-  end
+      @logger.info("Starting tcp output listener", :address => "#{@host}:#{@port}")
+      @server_socket = TCPServer.new(@host, @port)
+      @client_threads = []
 
-  public
-  def flush(events, final=false)
-    # Send the event over http.
-    request = Net::HTTP::Post.new(@uri.path)
-    request.body = events.inspect
-    request.add_field("Content-Type", 'application/json')
-    response = @client.request(request)
-    if response.is_a?(Net::HTTPSuccess)
-      @logger.debug("Event sent to Logmatic OK!")
+      @accept_thread = Thread.new(@server_socket) do |server_socket|
+        loop do
+          client_thread = Thread.start(server_socket.accept) do |client_socket|
+            client = Client.new(client_socket, @logger)
+            Thread.current[:client] = client
+            client.run
+          end
+          @client_threads << client_thread
+        end
+      end
+
+      @codec.on_event do |event, payload|
+        @client_threads.each do |client_thread|
+          client_thread[:client].write(payload)
+        end
+        @client_threads.reject! {|t| !t.alive? }
+      end
     else
-      @logger.warn("HTTP error", :error => response.error!)
+      client_socket = nil
+      @codec.on_event do |event, payload|
+        begin
+          client_socket = connect unless client_socket
+          r,w,e = IO.select([client_socket], [client_socket], [client_socket], nil)
+          # don't expect any reads, but a readable socket might
+          # mean the remote end closed, so read it and throw it away.
+          # we'll get an EOFError if it happens.
+          client_socket.sysread(16384) if r.any?
+
+          # Now send the payload
+          @logger.debug(@key);
+          client_socket.syswrite(@key + " " + payload+"\n") if w.any?
+        rescue => e
+          @logger.warn("tcp output exception", :host => @host, :port => @port,
+                       :exception => e, :backtrace => e.backtrace)
+          client_socket.close rescue nil
+          client_socket = nil
+          sleep @reconnect_interval
+          retry
+        end
+      end
+    end
+  end # def register
+
+  private
+  def connect
+    Stud::try do
+      return TCPSocket.new(@host, @port)
     end
+  end # def connect
+
+  private
+  def server?
+    @mode == "server"
+  end # def server?
+
+  public
+  def receive(event)
+
+
+    #if @message_format
+      #output = event.sprintf(@message_format) + "\n"
+    #else
+      #output = event.to_hash.to_json + "\n"
+    #end
+
+    @codec.encode(event)
   end # def receive
-end # class LogStash::Outputs::LogmaticBatch
+end # class LogStash::Outputs::Tcp

data/lib/logstash/outputs/logmatic_https.rb

@@ -0,0 +1,65 @@
+# encoding: utf-8
+require "logstash/outputs/base"
+require "logstash/namespace"
+require "stud/buffer"
+
+# Ship log from logstash straight to Logmatic
+#
+# To use this you will need a valid Logmatic API Key
+class LogStash::Outputs::LogmaticHttpBatch < LogStash::Outputs::Base
+  include Stud::Buffer
+
+  config_name "logmatic_http"
+  milestone 2
+
+  default :codec, "json"
+
+  # The Logmatic api key
+  # You can find it in the 'Account' section in the Logmatic interface.
+  config :key, :validate => :string, :required => true
+
+  # How many events to queue before flushing to Logmatic
+  # prior to schedule set in @timeframe
+  config :queue_size, :validate => :number, :default => 10
+
+  # How often (in seconds) to flush queued events to Logmatic
+  config :timeframe, :validate => :number, :default => 10
+
+  public
+  def register
+    require "net/https"
+    require "uri"
+    @url = "https://api.logmatic.io/v1/input/#{@key}"
+    @uri = URI.parse(@url)
+    @client = Net::HTTP.new(@uri.host, @uri.port)
+    @client.use_ssl = true
+    @client.verify_mode = OpenSSL::SSL::VERIFY_NONE
+    @logger.debug("Logmatic URL", :url => @url)
+
+    buffer_initialize(
+      :max_items => @queue_size,
+      :max_interval => @timeframe,
+      :logger => @logger
+    )
+  end
+
+  public
+  def receive(event)
+    return unless output?(event)
+    buffer_receive(event.to_json)
+  end
+
+  public
+  def flush(events, final=false)
+    # Send the event over http.
+    request = Net::HTTP::Post.new(@uri.path)
+    request.body = events.inspect
+    request.add_field("Content-Type", 'application/json')
+    response = @client.request(request)
+    if response.is_a?(Net::HTTPSuccess)
+      @logger.debug("Event sent to Logmatic OK!")
+    else
+      @logger.warn("HTTP error", :error => response.error!)
+    end
+  end # def receive
+end # class LogStash::Outputs::LogmaticBatch

data/logstash-output-logmatic.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name = 'logstash-output-logmatic'
-  s.version         = "0.1.1"
+  s.version         = "0.1.2"
   s.licenses = ["GNU 2.0"]
   s.summary = "Logmatic Output"
   s.description = "Logmatic output for Logstash"

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: logstash-output-logmatic
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 0.1.2
 platform: ruby
 authors:
 - Giovanni CLEMENT
@@ -9,7 +9,7 @@ authors:
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2015-10-07 00:00:00.000000000 Z
+date: 2015-10-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -67,6 +67,7 @@ files:
 - NOTICE.TXT
 - README.md
 - lib/logstash/outputs/logmatic.rb
+- lib/logstash/outputs/logmatic_https.rb
 - logstash-output-logmatic.gemspec
 - spec/outputs/logmatic.rb
 homepage: ''