fluent-plugin-logmatic 0.9.0 → 0.9.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 49374af53808bfdd30a59f976c71fb1e161ceecb
-  data.tar.gz: 776691802f58b23f15ee374916b7e9b594796253
+  metadata.gz: 5a30e2c66cf206b595f59ba1be9bbd34a0c3ac26
+  data.tar.gz: 3a426bb92f291ade3dbbe01d669c7f68c97937a7
 SHA512:
-  metadata.gz: 845f4a9a04e4ee16e8b3a2386cc452b990aa437c5be54eb9960ed8ea1a2384ac12770b09cd3caace07f31ac74da5c984bfc3b291821ac35dc5fb809a5945fcb8
-  data.tar.gz: 7c177051f77ae73a8f4f58912763a69547cdbe362e7db1e2c43c3aedeafc67347e880454e9e68e7caed6d82f0b4dc192bc83961d72a587dfc737854ecf320984
+  metadata.gz: dbc9ec516620c079c5c00ad5948e398fa814d4982f15a8d43909d8e19d3af7cb32205ac6922ba4aa93ce6f3c61cb56114840f345590f7b1c89e1d5bb7ef9fd07
+  data.tar.gz: 30d38f4691524a4641f23ea69db610ce9a32ee6d4ce0260eed1a8cc00cddbd54e8ca4b724f347b4f4886a400ef5ed00840058b168a95e9cc96af49e1dd76bad1

data/README.md

@@ -1,5 +1,5 @@
 # Fluentd output plugin for Logmatic.io.
-Link to the [Logmatic.io documentation](http://doc.logmatic.io/docs/using-fluentd)*
+Link to the [Logmatic.io documentation](http://doc.logmatic.io/docs/using-fluentd)
 
 
 It mainly contains a proper JSON formatter and a socket handler that
@@ -17,6 +17,7 @@ To add the plugin to your fluentd agent, use the following command:
 
 To match events and send them to logmatic.io, simply add the following code to your configuration file.
 
+TCP example
 ```xml
 # Match events tagged with "logmatic.**" and
 # send them to Logmatic.io
@@ -32,6 +33,23 @@ To match events and send them to logmatic.io, simply add the following code to y
   
 </match>
 
+```
+HTTP example
+```xml
+# Match events tagged with "logmatic.**" and
+# send them to Logmatic.io
+<match logmatic.**>
+
+  @type logmatic_http
+  @id awesome_agent
+  api_key <your_api_key>
+
+  # Optional
+  include_tag_key true
+  tag_key 'tag'
+  
+</match>
+
 ```
 
 After a restart of FluentD, any child events tagged with `logmatic` are shipped to your plateform.

data/fluent-plugin-logmatic.gemspec

@@ -4,7 +4,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |spec|
   spec.name          = "fluent-plugin-logmatic"
-  spec.version       = "0.9.0"
+  spec.version       = "0.9.1"
   spec.authors       = ["Logmatic support team"]
   spec.email         = ["support@logmatic.io"]
   spec.summary       = "Logmatic output plugin for Fluent event collector"
@@ -18,4 +18,5 @@ Gem::Specification.new do |spec|
 
   spec.add_development_dependency "bundler", "~> 1.5"
   spec.add_development_dependency "rake"
+  spec.add_development_dependency "yajl-ruby", "~> 1.2"
 end

data/lib/fluent/plugin/out_logmatic.rb

@@ -1,5 +1,6 @@
 require 'socket'
 require 'openssl'
+require 'yajl'
 
 class Fluent::LogmaticOutput < Fluent::BufferedOutput
   class ConnectionFailure < StandardError; end
@@ -75,12 +76,13 @@ class Fluent::LogmaticOutput < Fluent::BufferedOutput
     messages = Array.new
     chunk.msgpack_each do |tag, record|
       next unless record.is_a? Hash
-      next unless @use_json or record.has_key? "message"
+      next unless record.has_key? "message"
+
       if @include_tag_key
         record[@tag_key] = tag
       end
       if @use_json
-        messages.push "#{api_key} " + record.to_json + "\n"
+        messages.push "#{api_key} " + Yajl.dump(record) + "\n"
       else
         messages.push "#{api_key} " + record["message"].rstrip() + "\n"
       end

data/lib/fluent/plugin/out_logmatic_http.rb

@@ -1,3 +1,8 @@
+require 'net/http'
+require 'net/https'
+require 'uri'
+require 'yajl'
+
 class Fluent::LogmaticOutput < Fluent::BufferedOutput
   class ConnectionFailure < StandardError;
   end
@@ -20,9 +25,6 @@ class Fluent::LogmaticOutput < Fluent::BufferedOutput
 
   def initialize
     super
-    require 'net/http'
-    require 'net/https'
-    require 'uri'
   end
 
   def configure(conf)
@@ -56,45 +58,58 @@ class Fluent::LogmaticOutput < Fluent::BufferedOutput
 
     # Pack messages
     chunk.msgpack_each do |tag, record|
+
+      log.trace("New chunk received: #{record}, #{tag}")
       next unless record.is_a? Hash
-      next unless @use_json or record.has_key? "message"
+      next unless record.has_key? "message"
+
 
       if @include_tag_key
         record[@tag_key] = tag
       end
-      messages.push record.to_json
+
+      log.trace("Json message: #{Yajl.dump(record)}")
+      messages.push Yajl.dump(record)
     end
 
     # Send them
     log.trace("Sending #{messages.length} messages")
     retries = 0
-    begin
+    force_retry = false
+
+    if (messages.length > 0)
 
-      req = Net::HTTP::Post.new(@uri.path)
-      req['Content-Type'] = 'application/json'
-      req.body = data
-      log.trace("Posting data")
-      res = @https.request(messages.to_json)
-      log.trace("Status code: #{res.code}")
+      log.trace("Full message body: #{Yajl.dump(messages)}")
+      begin
 
-      if (res.code != 0 && res.code != 400)
-        if retries < @max_retries || max_retries == -1
+        force_retry = false
+        req = Net::HTTP::Post.new(@uri.path)
+        req['Content-Type'] = 'application/json'
+        req.body = Yajl.dump(messages)
+        log.trace("Posting data")
+        res = @https.request(req)
+        log.trace("Status code: #{res.code}")
 
-          a_couple_of_seconds = retries ** 2
-          a_couple_of_seconds = 30 unless a_couple_of_seconds < 30
-          retries += 1
-          log.warn "Could not push logs to Logmatic, attempt=#{retries} max_attempts=#{max_retries} wait=#{a_couple_of_seconds}s error=#{res.code}"
+        if (res.code.to_i != 0 && res.code.to_i != 200)
+          if retries < @max_retries || max_retries == -1
 
-          sleep a_couple_of_seconds
+            a_couple_of_seconds = retries ** 2
+            a_couple_of_seconds = 30 unless a_couple_of_seconds < 30
+            retries += 1
+            log.warn "Could not push logs to Logmatic, attempt=#{retries} max_attempts=#{max_retries} wait=#{a_couple_of_seconds}s error=#{res.code}"
 
-          raise "Status code: #{res.code}"
+            sleep a_couple_of_seconds
+            force_retry = true
+            raise "Status code: #{res.code}"
 
+          end
         end
+      rescue => e
+        # Handle some failures
+        log.error("Error while sending data. Making a new attempt. Error: #{e}")
+        retry if force_retry
+        raise ConnectionFailure, "Could not push logs to Logmatic after #{retries} retries, #{e.message}"
       end
-    rescue => e
-      # Handle some failures
-      retry
-      raise ConnectionFailure, "Could not push logs to Logmatic after #{retries} retries, #{e.message}"
     end
   end
 

metadata

@@ -1,43 +1,57 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-logmatic
 version: !ruby/object:Gem::Version
-  version: 0.9.0
+  version: 0.9.1
 platform: ruby
 authors:
 - Logmatic support team
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-05-03 00:00:00.000000000 Z
+date: 2017-06-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ~>
       - !ruby/object:Gem::Version
         version: '1.5'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ~>
       - !ruby/object:Gem::Version
         version: '1.5'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: yajl-ruby
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.2'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.2'
 description: 
 email:
 - support@logmatic.io
@@ -45,7 +59,7 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
-- ".gitignore"
+- .gitignore
 - Gemfile
 - LICENSE
 - README.md
@@ -63,17 +77,17 @@ require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">="
+  - - '>='
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">="
+  - - '>='
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.6.8
+rubygems_version: 2.0.14.1
 signing_key: 
 specification_version: 4
 summary: Logmatic output plugin for Fluent event collector