fluent-plugin-simple-logentries 0.0.3 → 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: cfc43c1522f9777ae9209c1b056fb1925d3642be
4
- data.tar.gz: 69f7482845701c7d8aec0942403d9f38165817e3
3
+ metadata.gz: 06e733cb821b8f88bf7120c955bc9b4a3e448d35
4
+ data.tar.gz: 20a8207684c939d30320cfd934090fd30ed16df9
5
5
  SHA512:
6
- metadata.gz: df1a9f26b4473c788693483903043cfad2581dad703638e209ca5d1b5d0bc815f741c9eb72891435464af6a5a64172f5bcd87cd9e3315263c24456170a7ba6db
7
- data.tar.gz: 68bdf4922f142259d1d041c3044655e6f9465c449f13f472b5aa50e085ffc44a2676eb30c807c6819861e0cb675aa60d9d6e692f5bf6176218414bc6ad72b128
6
+ metadata.gz: e2f8de1f82ab72e5395587bf527a11e17d31eadb5f030949b53613dcc9779209c8501b22be75e41e29cd0b7ab7a7b65402697e166223e9d7ec3a7fbc29d5270c
7
+ data.tar.gz: 1507e874c7e97992468235b934477fb9ad63c07fbe64e4f4f4ac00052411e2f059b2ca888567b87127d352f8c4c3ba40f92517d780cc64f8d519eacf8fe56137
data/.gitignore ADDED
@@ -0,0 +1,18 @@
1
+ *.gem
2
+ *.rbc
3
+ .bundle
4
+ .config
5
+ .yardoc
6
+ .DS_Store
7
+ Gemfile.lock
8
+ InstalledFiles
9
+ _yardoc
10
+ coverage
11
+ doc/
12
+ lib/bundler/man
13
+ pkg
14
+ rdoc
15
+ spec/reports
16
+ test/tmp
17
+ test/version_tmp
18
+ tmp
data/Gemfile CHANGED
@@ -1,2 +1,3 @@
1
1
  source 'https://rubygems.org'
2
+ gem 'fluentd', '~> 0.12.15'
2
3
  gemspec
@@ -4,7 +4,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
4
4
 
5
5
  Gem::Specification.new do |spec|
6
6
  spec.name = "fluent-plugin-simple-logentries"
7
- spec.version = "0.0.3"
7
+ spec.version = "0.1.0"
8
8
  spec.description = "Push fluent events to Logentries"
9
9
  spec.authors = ["sowawa"]
10
10
  spec.email = ["kesiuke.sogawa@gmail.com"]
@@ -1,6 +1,7 @@
1
1
  require 'socket'
2
2
  require 'json'
3
3
  require 'openssl'
4
+ require 'securerandom'
4
5
 
5
6
  class Fluent::SimpleLogentriesOutput < Fluent::BufferedOutput
6
7
  class ConnectionFailure < StandardError; end
@@ -14,6 +15,8 @@ class Fluent::SimpleLogentriesOutput < Fluent::BufferedOutput
14
15
  config_param :token, :string
15
16
  SSL_HOST = "api.logentries.com"
16
17
  NO_SSL_HOST = "data.logentries.com"
18
+ MAX_ENTRY_SIZE = 8192
19
+ SPLITED_ENTRY_SIZE = MAX_ENTRY_SIZE - 256
17
20
 
18
21
  def configure(conf)
19
22
  super
@@ -54,17 +57,40 @@ class Fluent::SimpleLogentriesOutput < Fluent::BufferedOutput
54
57
  def write(chunk)
55
58
  chunk.msgpack_each do |tag, record|
56
59
  if record.is_a? Hash
57
- send_logentries(@token,
58
- JSON.generate(
59
- @append_tag ? record.merge({tag: tag}) : record))
60
+ send_logentries(tag, record)
60
61
  end
61
62
  end
62
63
  end
63
64
 
64
- def send_logentries(token, data)
65
+ def send_logentries(tag, record)
66
+ data = if @append_tag
67
+ record.merge({tag: tag})
68
+ else
69
+ record
70
+ end
71
+ jsonfied = JSON.generate(data)
72
+ if (data.member?(:messages) || data.member?('messages')) && jsonfied.length > MAX_ENTRY_SIZE
73
+ identifyer = SecureRandom.uuid
74
+ messages = if data.member?(:messages)
75
+ data[:messages]
76
+ elsif data.member?('messages')
77
+ data['messages']
78
+ end
79
+ data.delete(:messages) && data.delete('messages')
80
+ ([data] + split_messages(messages).map{|i| {messages: i}} ).each_with_index { |item, idx|
81
+ push(JSON.generate({sequence: idx, identifyer: identifyer}.merge(item)))
82
+ }
83
+ else
84
+ push(jsonfied)
85
+ end
86
+ rescue => e
87
+ log.warn "Could not push logs to Logentries. #{e.message}"
88
+ end
89
+
90
+ def push(data)
65
91
  retries = 0
66
92
  begin
67
- client.write("#{token} #{data} \n")
93
+ client.write("#{@token} #{data} \n")
68
94
  rescue Errno::ECONNREFUSED, Errno::ETIMEDOUT => e
69
95
  if retries < @max_retries
70
96
  retries += 1
@@ -74,9 +100,15 @@ class Fluent::SimpleLogentriesOutput < Fluent::BufferedOutput
74
100
  retry
75
101
  end
76
102
  raise ConnectionFailure, "Could not push logs to Logentries after #{retries} retries. #{e.message}"
77
- rescue Errno::EMSGSIZE
103
+ rescue Errno::EMSGSIZE => e
78
104
  log.warn "Could not push logs to Logentries. #{e.message}"
79
105
  end
80
106
  end
81
107
 
108
+ def split_messages(messages)
109
+ str_length = messages.length
110
+ return [messages] if SPLITED_ENTRY_SIZE > str_length
111
+ split_messages(messages[0..str_length/2]) +
112
+ split_messages(messages[(str_length/2)+1..str_length])
113
+ end
82
114
  end
data/test.conf ADDED
@@ -0,0 +1,8 @@
1
+ <source>
2
+ type forward
3
+ </source>
4
+ <match rails.**>
5
+ type simple-logentries
6
+ token 08f1b304-7b73-49d3-829a-49ab3b60ca37
7
+ flush_interval 2s
8
+ </match>
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: fluent-plugin-simple-logentries
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.0.3
4
+ version: 0.1.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - sowawa
@@ -45,9 +45,11 @@ executables: []
45
45
  extensions: []
46
46
  extra_rdoc_files: []
47
47
  files:
48
+ - ".gitignore"
48
49
  - Gemfile
49
50
  - fluent-plugin-simple-logentries.gemspec
50
51
  - lib/fluent/plugin/out_simple-logentries.rb
52
+ - test.conf
51
53
  homepage: https://github.com/sowawa/fluent-plugin-simple-logentries
52
54
  licenses:
53
55
  - MIT