fluent-plugin-simple-logentries 0.0.3 → 0.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: cfc43c1522f9777ae9209c1b056fb1925d3642be
-  data.tar.gz: 69f7482845701c7d8aec0942403d9f38165817e3
+  metadata.gz: 06e733cb821b8f88bf7120c955bc9b4a3e448d35
+  data.tar.gz: 20a8207684c939d30320cfd934090fd30ed16df9
 SHA512:
-  metadata.gz: df1a9f26b4473c788693483903043cfad2581dad703638e209ca5d1b5d0bc815f741c9eb72891435464af6a5a64172f5bcd87cd9e3315263c24456170a7ba6db
-  data.tar.gz: 68bdf4922f142259d1d041c3044655e6f9465c449f13f472b5aa50e085ffc44a2676eb30c807c6819861e0cb675aa60d9d6e692f5bf6176218414bc6ad72b128
+  metadata.gz: e2f8de1f82ab72e5395587bf527a11e17d31eadb5f030949b53613dcc9779209c8501b22be75e41e29cd0b7ab7a7b65402697e166223e9d7ec3a7fbc29d5270c
+  data.tar.gz: 1507e874c7e97992468235b934477fb9ad63c07fbe64e4f4f4ac00052411e2f059b2ca888567b87127d352f8c4c3ba40f92517d780cc64f8d519eacf8fe56137

data/.gitignore

@@ -0,0 +1,18 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+.DS_Store
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp

data/Gemfile

@@ -1,2 +1,3 @@
 source 'https://rubygems.org'
+gem 'fluentd', '~> 0.12.15'
 gemspec

data/fluent-plugin-simple-logentries.gemspec

@@ -4,7 +4,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |spec|
   spec.name          = "fluent-plugin-simple-logentries"
-  spec.version       = "0.0.3"
+  spec.version       = "0.1.0"
   spec.description   = "Push fluent events to Logentries"
   spec.authors       = ["sowawa"]
   spec.email         = ["kesiuke.sogawa@gmail.com"]

data/lib/fluent/plugin/out_simple-logentries.rb

@@ -1,6 +1,7 @@
 require 'socket'
 require 'json'
 require 'openssl'
+require 'securerandom'
 
 class Fluent::SimpleLogentriesOutput < Fluent::BufferedOutput
   class ConnectionFailure < StandardError; end
@@ -14,6 +15,8 @@ class Fluent::SimpleLogentriesOutput < Fluent::BufferedOutput
   config_param :token,          :string
   SSL_HOST    = "api.logentries.com"
   NO_SSL_HOST = "data.logentries.com"
+  MAX_ENTRY_SIZE = 8192
+  SPLITED_ENTRY_SIZE = MAX_ENTRY_SIZE - 256
 
   def configure(conf)
     super
@@ -54,17 +57,40 @@ class Fluent::SimpleLogentriesOutput < Fluent::BufferedOutput
   def write(chunk)
     chunk.msgpack_each do |tag, record|
       if record.is_a? Hash
-        send_logentries(@token,
-          JSON.generate(
-            @append_tag ? record.merge({tag: tag}) : record))
+        send_logentries(tag, record)
       end
     end
   end
 
-  def send_logentries(token, data)
+  def send_logentries(tag, record)
+    data = if @append_tag
+             record.merge({tag: tag})
+           else
+             record
+           end
+    jsonfied = JSON.generate(data)
+    if (data.member?(:messages) || data.member?('messages')) && jsonfied.length > MAX_ENTRY_SIZE
+      identifyer = SecureRandom.uuid
+      messages = if data.member?(:messages)
+                   data[:messages]
+                 elsif data.member?('messages')
+                   data['messages']
+                 end
+      data.delete(:messages) && data.delete('messages')
+      ([data] + split_messages(messages).map{|i| {messages: i}} ).each_with_index { |item, idx|
+        push(JSON.generate({sequence: idx, identifyer: identifyer}.merge(item)))
+      }
+    else
+      push(jsonfied)
+    end
+  rescue => e
+    log.warn "Could not push logs to Logentries. #{e.message}"
+  end
+
+  def push(data)
     retries = 0
     begin
-      client.write("#{token} #{data} \n")
+      client.write("#{@token} #{data} \n")
     rescue Errno::ECONNREFUSED, Errno::ETIMEDOUT => e
       if retries < @max_retries
         retries += 1
@@ -74,9 +100,15 @@ class Fluent::SimpleLogentriesOutput < Fluent::BufferedOutput
         retry
       end
       raise ConnectionFailure, "Could not push logs to Logentries after #{retries} retries. #{e.message}"
-    rescue Errno::EMSGSIZE
+    rescue Errno::EMSGSIZE => e
       log.warn "Could not push logs to Logentries. #{e.message}"
     end
   end
 
+  def split_messages(messages)
+    str_length = messages.length
+    return [messages] if SPLITED_ENTRY_SIZE > str_length
+    split_messages(messages[0..str_length/2]) +
+      split_messages(messages[(str_length/2)+1..str_length])
+  end
 end

data/test.conf

@@ -0,0 +1,8 @@
+<source>
+  type forward
+</source>
+<match rails.**>
+  type simple-logentries
+  token 08f1b304-7b73-49d3-829a-49ab3b60ca37
+  flush_interval 2s
+</match>

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-simple-logentries
 version: !ruby/object:Gem::Version
-  version: 0.0.3
+  version: 0.1.0
 platform: ruby
 authors:
 - sowawa
@@ -45,9 +45,11 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
+- ".gitignore"
 - Gemfile
 - fluent-plugin-simple-logentries.gemspec
 - lib/fluent/plugin/out_simple-logentries.rb
+- test.conf
 homepage: https://github.com/sowawa/fluent-plugin-simple-logentries
 licenses:
 - MIT