fluent-logger 0.7.1 → 0.10.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +5 -5
- data/.github/workflows/linux.yml +28 -0
- data/ChangeLog +30 -1
- data/Gemfile +7 -6
- data/README.md +126 -9
- data/fluent-logger.gemspec +5 -17
- data/lib/fluent/logger/fluent_logger.rb +101 -15
- data/lib/fluent/logger/version.rb +3 -5
- data/spec/fluent_logger_spec.rb +132 -0
- data/spec/support/dummy_fluentd.rb +18 -2
- metadata +12 -72
- data/.travis.yml +0 -34
- data/VERSION +0 -1
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
|
-
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
2
|
+
SHA256:
|
|
3
|
+
metadata.gz: 35434f35801a28fb88c9802cfd213caa4a9a052f087241768f676b0f7c67958a
|
|
4
|
+
data.tar.gz: cf2b26bbf41d008c9c3e462a0dcd5e2c23cfbdbc9c08227d31454d4b171abb83
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 899bccf2efee5b4c230155e1ef6beb9fa4b1f06b843fb9abd5f87630246f1f112b75200fa0d431de3aeba5e9dbf13e55a97fa3b57945112530cdda57288d1cc7
|
|
7
|
+
data.tar.gz: 8bebd7215231cef3379b5a6e9ccdc72407b0198874d44860033056cc97600dce811a5929cf13533bd42e1954c24fe01b9de5c27037a57a6b4354b6adbe41e393
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
name: linux
|
|
2
|
+
on:
|
|
3
|
+
- push
|
|
4
|
+
- pull_request
|
|
5
|
+
jobs:
|
|
6
|
+
build:
|
|
7
|
+
runs-on: ${{ matrix.os }}
|
|
8
|
+
strategy:
|
|
9
|
+
fail-fast: false
|
|
10
|
+
matrix:
|
|
11
|
+
ruby:
|
|
12
|
+
- '3.2'
|
|
13
|
+
- '3.3'
|
|
14
|
+
- '3.4'
|
|
15
|
+
os:
|
|
16
|
+
- ubuntu-latest
|
|
17
|
+
name: Ruby ${{ matrix.ruby }} unit testing on ${{ matrix.os }}
|
|
18
|
+
steps:
|
|
19
|
+
- uses: actions/checkout@v4
|
|
20
|
+
- uses: ruby/setup-ruby@v1
|
|
21
|
+
with:
|
|
22
|
+
ruby-version: ${{ matrix.ruby }}
|
|
23
|
+
bundler-cache: true
|
|
24
|
+
- name: unit testing
|
|
25
|
+
env:
|
|
26
|
+
CI: true
|
|
27
|
+
run: |
|
|
28
|
+
bundle exec rake spec
|
data/ChangeLog
CHANGED
|
@@ -1,3 +1,33 @@
|
|
|
1
|
+
Release 0.10.0 - 2025/09/04
|
|
2
|
+
|
|
3
|
+
* Add timeout option (#104)
|
|
4
|
+
* gemspec: add logger gem as runtime dependency
|
|
5
|
+
|
|
6
|
+
Release 0.9.1 - 2023/12/20
|
|
7
|
+
|
|
8
|
+
* Fix invalid message polluting subsequent message
|
|
9
|
+
|
|
10
|
+
Release 0.9.0 - 2020/09/04
|
|
11
|
+
|
|
12
|
+
* FluentLogger supports TLS
|
|
13
|
+
|
|
14
|
+
Release 0.8.2 - 2019/08/21
|
|
15
|
+
|
|
16
|
+
* Block timeout during IO#write to avoid writing invalid bytes
|
|
17
|
+
* Fluent::Logger::EventTime#to_json returns String
|
|
18
|
+
|
|
19
|
+
Release 0.8.1 - 2019/05/30
|
|
20
|
+
|
|
21
|
+
* Improve non-blocking write handling
|
|
22
|
+
|
|
23
|
+
Release 0.8.0 - 2019/01/25
|
|
24
|
+
|
|
25
|
+
* Add use_nonblock and wait_writeable parameters
|
|
26
|
+
|
|
27
|
+
Release 0.7.2 - 2018/01/04
|
|
28
|
+
|
|
29
|
+
* Fix EventTime for nanosecond_precision
|
|
30
|
+
|
|
1
31
|
Release 0.7.1 - 2017/04/19
|
|
2
32
|
|
|
3
33
|
* Fix packer clear bug for non-msgpackable data
|
|
@@ -122,4 +152,3 @@ Release 0.2.0 - 2011/08/05
|
|
|
122
152
|
Release 0.1.0 - 2011/08/04
|
|
123
153
|
|
|
124
154
|
* First release
|
|
125
|
-
|
data/Gemfile
CHANGED
|
@@ -1,11 +1,12 @@
|
|
|
1
|
-
|
|
2
1
|
source 'https://rubygems.org/'
|
|
3
2
|
|
|
4
|
-
gem "fluentd", :github => 'fluent/fluentd'
|
|
5
|
-
|
|
6
3
|
gemspec
|
|
7
4
|
|
|
8
|
-
gem "
|
|
5
|
+
gem "rake", ">= 0.9.2"
|
|
6
|
+
gem "rspec", ">= 3.0.0"
|
|
7
|
+
gem "rspec-its", ">= 1.1.0"
|
|
8
|
+
gem "timecop", ">= 0.3.0"
|
|
9
|
+
gem "fluentd"
|
|
10
|
+
gem "test-unit"
|
|
11
|
+
gem "simplecov", ">= 0.5.4", require: false
|
|
9
12
|
gem "simplecov-vim"
|
|
10
|
-
|
|
11
|
-
gem 'test-unit'
|
data/README.md
CHANGED
|
@@ -1,9 +1,25 @@
|
|
|
1
1
|
# Fluent logger
|
|
2
2
|
|
|
3
|
-
[](https://github.com/fluent/fluent-logger-ruby/actions/workflows/linux.yml)
|
|
4
4
|
|
|
5
5
|
A structured event logger
|
|
6
6
|
|
|
7
|
+
## Installation
|
|
8
|
+
|
|
9
|
+
Add this line to your application's Gemfile:
|
|
10
|
+
|
|
11
|
+
```ruby
|
|
12
|
+
gem 'fluent-logger'
|
|
13
|
+
```
|
|
14
|
+
|
|
15
|
+
And then execute:
|
|
16
|
+
|
|
17
|
+
$ bundle install
|
|
18
|
+
|
|
19
|
+
Or install it yourself as:
|
|
20
|
+
|
|
21
|
+
$ gem install fluent-logger
|
|
22
|
+
|
|
7
23
|
## Examples
|
|
8
24
|
|
|
9
25
|
### Simple
|
|
@@ -11,6 +27,7 @@ A structured event logger
|
|
|
11
27
|
```ruby
|
|
12
28
|
require 'fluent-logger'
|
|
13
29
|
|
|
30
|
+
# API: FluentLogger.new(tag_prefix, options)
|
|
14
31
|
log = Fluent::Logger::FluentLogger.new(nil, :host => 'localhost', :port => 24224)
|
|
15
32
|
unless log.post("myapp.access", {"agent" => "foo"})
|
|
16
33
|
p log.last_error # You can get last error object via last_error method
|
|
@@ -26,32 +43,132 @@ require 'fluent-logger'
|
|
|
26
43
|
|
|
27
44
|
log = Fluent::Logger::FluentLogger.new(nil, :socket_path => "/tmp/fluent.sock")
|
|
28
45
|
unless log.post("myapp.access", {"agent" => "foo"})
|
|
46
|
+
# Passed records are stored into logger's internal buffer so don't re-post same event.
|
|
29
47
|
p log.last_error # You can get last error object via last_error method
|
|
30
48
|
end
|
|
31
49
|
|
|
32
50
|
# output: myapp.access {"agent":"foo"}
|
|
33
51
|
```
|
|
34
52
|
|
|
35
|
-
###
|
|
53
|
+
### Tag prefix
|
|
36
54
|
```ruby
|
|
37
55
|
require 'fluent-logger'
|
|
38
56
|
|
|
39
|
-
Fluent::Logger::FluentLogger.
|
|
40
|
-
|
|
57
|
+
log = Fluent::Logger::FluentLogger.new('myapp', :host => 'localhost', :port => 24224)
|
|
58
|
+
log.post("access", {"agent" => "foo"})
|
|
41
59
|
|
|
42
60
|
# output: myapp.access {"agent":"foo"}
|
|
43
61
|
```
|
|
44
62
|
|
|
45
|
-
###
|
|
63
|
+
### Nonblocking write
|
|
64
|
+
|
|
46
65
|
```ruby
|
|
47
66
|
require 'fluent-logger'
|
|
48
67
|
|
|
49
|
-
log = Fluent::Logger::FluentLogger.new(
|
|
50
|
-
|
|
68
|
+
log = Fluent::Logger::FluentLogger.new(nil, :host => 'localhost', :port => 24224, :use_nonblock => true, :wait_writeable => false)
|
|
69
|
+
# When wait_writeable is false
|
|
70
|
+
begin
|
|
71
|
+
log.post("myapp.access", {"agent" => "foo"})
|
|
72
|
+
rescue IO::EAGAINWaitWritable => e
|
|
73
|
+
# wait code for avoding "Resource temporarily unavailable"
|
|
74
|
+
# Passed records are stored into logger's internal buffer so don't re-post same event.
|
|
75
|
+
end
|
|
76
|
+
|
|
77
|
+
# When wait_writeable is true
|
|
78
|
+
unless log.post("myapp.access", {"agent" => "foo"})
|
|
79
|
+
# same as other example
|
|
80
|
+
end
|
|
81
|
+
|
|
82
|
+
# output: myapp.access {"agent":"foo"}
|
|
83
|
+
```
|
|
84
|
+
|
|
85
|
+
### TLS setting
|
|
86
|
+
|
|
87
|
+
```ruby
|
|
88
|
+
require 'fluent-logger'
|
|
89
|
+
|
|
90
|
+
tls_opts = {
|
|
91
|
+
:ca => '/path/to/cacert.pem',
|
|
92
|
+
:cert => '/path/to/client-cert.pem',
|
|
93
|
+
:key => '/path/to/client-key.pem',
|
|
94
|
+
:key_passphrase => 'test'
|
|
95
|
+
}
|
|
96
|
+
log = Fluent::Logger::FluentLogger.new(nil, :host => 'localhost', :port => 24224, :tls_options => tls_opts)
|
|
97
|
+
```
|
|
98
|
+
|
|
99
|
+
`in_forward` config example:
|
|
100
|
+
|
|
101
|
+
```
|
|
102
|
+
<source>
|
|
103
|
+
@type forward
|
|
104
|
+
<transport tls>
|
|
105
|
+
version TLS1_2
|
|
106
|
+
ca_path /path/to/cacert.pem
|
|
107
|
+
cert_path /path/to/server-cert.pem
|
|
108
|
+
private_key_path /path/to/server-key.pem
|
|
109
|
+
private_key_passphrase test
|
|
110
|
+
client_cert_auth true
|
|
111
|
+
</transport>
|
|
112
|
+
</source>
|
|
113
|
+
```
|
|
114
|
+
|
|
115
|
+
### Singleton
|
|
116
|
+
```ruby
|
|
117
|
+
require 'fluent-logger'
|
|
118
|
+
|
|
119
|
+
Fluent::Logger::FluentLogger.open(nil, :host => 'localhost', :port => 24224)
|
|
120
|
+
Fluent::Logger.post("myapp.access", {"agent" => "foo"})
|
|
51
121
|
|
|
52
122
|
# output: myapp.access {"agent":"foo"}
|
|
53
123
|
```
|
|
54
124
|
|
|
125
|
+
### Logger options
|
|
126
|
+
|
|
127
|
+
#### host (String)
|
|
128
|
+
|
|
129
|
+
fluentd instance host
|
|
130
|
+
|
|
131
|
+
#### port (Integer)
|
|
132
|
+
|
|
133
|
+
fluentd instance port
|
|
134
|
+
|
|
135
|
+
#### socket_path (String)
|
|
136
|
+
|
|
137
|
+
If specified, fluentd uses unix domain socket instead of TCP.
|
|
138
|
+
|
|
139
|
+
#### nanosecond_precision (Bool)
|
|
140
|
+
|
|
141
|
+
Use nano second event time instead of epoch. See also "Tips" section.
|
|
142
|
+
|
|
143
|
+
#### use_nonblock (Bool)
|
|
144
|
+
|
|
145
|
+
Use nonblocking write(`IO#write_nonblock`) instead of normal write(`IO#write`). If `Logger#post` stuck on your environment, specify `true`. Default: `false`
|
|
146
|
+
|
|
147
|
+
#### wait_writeable (Bool)
|
|
148
|
+
|
|
149
|
+
If `false`, `Logger#post` raises an error when nonblocking write gets `EAGAIN` (i.e. `use_nonblock` must be `true`, otherwise this will have no effect). Default: `true`
|
|
150
|
+
|
|
151
|
+
#### timeout (Integer)
|
|
152
|
+
|
|
153
|
+
Specify a timeout in seconds for connecting. Default: `0` (Disabled)
|
|
154
|
+
|
|
155
|
+
#### buffer_overflow_handler (Proc)
|
|
156
|
+
|
|
157
|
+
Pass callback for handling buffer overflow with pending data. See "Buffer overflow" section.
|
|
158
|
+
|
|
159
|
+
#### tls_options (Hash)
|
|
160
|
+
|
|
161
|
+
Pass TLS related options.
|
|
162
|
+
|
|
163
|
+
- use_default_ca: Set `true` if you want to use default CA
|
|
164
|
+
- ca: CA file path
|
|
165
|
+
- cert: Certificate file path
|
|
166
|
+
- key: Private key file path
|
|
167
|
+
- key_passphrase: Private key passphrase
|
|
168
|
+
- version: TLS version. Default is `OpenSSL::SSL::TLS1_2_VERSION`
|
|
169
|
+
- ciphers: The list of cipher suites. Default is `ALL:!aNULL:!eNULL:!SSLv2`
|
|
170
|
+
- insecure: Set `true` when `in_forward` uses `insecure true`
|
|
171
|
+
|
|
55
172
|
### Standard ::Logger compatible interface
|
|
56
173
|
|
|
57
174
|
#### Example1
|
|
@@ -132,7 +249,7 @@ Fluent::Logger::NullLogger.open
|
|
|
132
249
|
|
|
133
250
|
To send events with nanosecond-precision time (Fluent 0.14 and up), specify `nanosecond_precision` to `FluentLogger` constructor.
|
|
134
251
|
|
|
135
|
-
```
|
|
252
|
+
```ruby
|
|
136
253
|
log = Fluent::Logger::FluentLogger.new(nil, :host => 'localhost', :port => 24224, :nanosecond_precision => true)
|
|
137
254
|
# Use nanosecond time instead
|
|
138
255
|
log.post("myapp.access", {"agent" => "foo"})
|
|
@@ -146,7 +263,7 @@ You can inject your own custom proc to handle buffer overflow in the event of co
|
|
|
146
263
|
Your proc must accept a single argument, which will be the internal buffer of messages from the logger. A typical use-case for this would be writing to disk or possibly writing to Redis.
|
|
147
264
|
|
|
148
265
|
##### Example
|
|
149
|
-
```
|
|
266
|
+
```ruby
|
|
150
267
|
class BufferOverflowHandler
|
|
151
268
|
attr_accessor :buffer
|
|
152
269
|
|
data/fluent-logger.gemspec
CHANGED
|
@@ -1,20 +1,10 @@
|
|
|
1
1
|
# encoding: utf-8
|
|
2
2
|
$:.push File.expand_path('../lib', __FILE__)
|
|
3
|
+
require 'fluent/logger/version'
|
|
3
4
|
|
|
4
5
|
Gem::Specification.new do |gem|
|
|
5
6
|
version_file = "lib/fluent/logger/version.rb"
|
|
6
|
-
version =
|
|
7
|
-
File.open(version_file, "w") {|f|
|
|
8
|
-
f.write <<EOF
|
|
9
|
-
module Fluent
|
|
10
|
-
module Logger
|
|
11
|
-
|
|
12
|
-
VERSION = '#{version}'
|
|
13
|
-
|
|
14
|
-
end
|
|
15
|
-
end
|
|
16
|
-
EOF
|
|
17
|
-
}
|
|
7
|
+
version = Fluent::Logger::VERSION
|
|
18
8
|
|
|
19
9
|
gem.name = %q{fluent-logger}
|
|
20
10
|
gem.version = version
|
|
@@ -29,11 +19,9 @@ EOF
|
|
|
29
19
|
gem.test_files = `git ls-files -- {test,spec,features}/*`.split("\n")
|
|
30
20
|
gem.executables = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
|
|
31
21
|
gem.require_paths = ['lib']
|
|
22
|
+
gem.license = "Apache-2.0"
|
|
32
23
|
|
|
33
24
|
gem.add_dependency "msgpack", ">= 1.0.0", "< 2"
|
|
34
|
-
gem
|
|
35
|
-
gem.
|
|
36
|
-
gem.add_development_dependency 'rspec-its', '>= 1.1.0'
|
|
37
|
-
gem.add_development_dependency 'simplecov', '>= 0.5.4'
|
|
38
|
-
gem.add_development_dependency 'timecop', '>= 0.3.0'
|
|
25
|
+
# logger gem that isn't default gems as of Ruby 3.5
|
|
26
|
+
gem.add_dependency "logger", "~> 1.6"
|
|
39
27
|
end
|
|
@@ -15,27 +15,31 @@
|
|
|
15
15
|
# See the License for the specific language governing permissions and
|
|
16
16
|
# limitations under the License.
|
|
17
17
|
#
|
|
18
|
+
require 'timeout'
|
|
18
19
|
require 'msgpack'
|
|
19
20
|
require 'socket'
|
|
21
|
+
require 'openssl'
|
|
20
22
|
require 'monitor'
|
|
21
23
|
require 'logger'
|
|
22
24
|
require 'json'
|
|
25
|
+
require 'timeout'
|
|
23
26
|
|
|
24
27
|
module Fluent
|
|
25
28
|
module Logger
|
|
26
29
|
class EventTime
|
|
27
30
|
TYPE = 0
|
|
28
31
|
|
|
29
|
-
def initialize(
|
|
30
|
-
@
|
|
32
|
+
def initialize(sec, nsec = 0)
|
|
33
|
+
@sec = sec
|
|
34
|
+
@nsec = nsec
|
|
31
35
|
end
|
|
32
36
|
|
|
33
37
|
def to_msgpack(io = nil)
|
|
34
|
-
@
|
|
38
|
+
@sec.to_msgpack(io)
|
|
35
39
|
end
|
|
36
40
|
|
|
37
41
|
def to_msgpack_ext
|
|
38
|
-
[@
|
|
42
|
+
[@sec, @nsec].pack('NN')
|
|
39
43
|
end
|
|
40
44
|
|
|
41
45
|
def self.from_msgpack_ext(data)
|
|
@@ -43,7 +47,7 @@ module Fluent
|
|
|
43
47
|
end
|
|
44
48
|
|
|
45
49
|
def to_json(*args)
|
|
46
|
-
@
|
|
50
|
+
@sec.to_s
|
|
47
51
|
end
|
|
48
52
|
end
|
|
49
53
|
|
|
@@ -63,7 +67,8 @@ module Fluent
|
|
|
63
67
|
|
|
64
68
|
options = {
|
|
65
69
|
:host => 'localhost',
|
|
66
|
-
:port => 24224
|
|
70
|
+
:port => 24224,
|
|
71
|
+
:use_nonblock => false
|
|
67
72
|
}
|
|
68
73
|
|
|
69
74
|
case args.first
|
|
@@ -80,6 +85,8 @@ module Fluent
|
|
|
80
85
|
@port = options[:port]
|
|
81
86
|
@socket_path = options[:socket_path]
|
|
82
87
|
@nanosecond_precision = options[:nanosecond_precision]
|
|
88
|
+
@use_nonblock = options[:use_nonblock]
|
|
89
|
+
@tls_options = options[:tls_options]
|
|
83
90
|
|
|
84
91
|
@factory = MessagePack::Factory.new
|
|
85
92
|
if @nanosecond_precision
|
|
@@ -95,8 +102,8 @@ module Fluent
|
|
|
95
102
|
@log_reconnect_error_threshold = options[:log_reconnect_error_threshold] || RECONNECT_WAIT_MAX_COUNT
|
|
96
103
|
|
|
97
104
|
@buffer_overflow_handler = options[:buffer_overflow_handler]
|
|
98
|
-
if
|
|
99
|
-
@logger = logger
|
|
105
|
+
if options[:logger]
|
|
106
|
+
@logger = options[:logger]
|
|
100
107
|
else
|
|
101
108
|
@logger = ::Logger.new(STDERR)
|
|
102
109
|
if options[:debug]
|
|
@@ -106,6 +113,11 @@ module Fluent
|
|
|
106
113
|
end
|
|
107
114
|
end
|
|
108
115
|
|
|
116
|
+
@wait_writeable = true
|
|
117
|
+
@wait_writeable = options[:wait_writeable] if options.key?(:wait_writeable)
|
|
118
|
+
|
|
119
|
+
@timeout = options[:timeout] || 0
|
|
120
|
+
|
|
109
121
|
@last_error = {}
|
|
110
122
|
|
|
111
123
|
begin
|
|
@@ -120,7 +132,6 @@ module Fluent
|
|
|
120
132
|
end
|
|
121
133
|
|
|
122
134
|
attr_accessor :limit, :logger, :log_reconnect_error_threshold
|
|
123
|
-
attr_reader :last_error
|
|
124
135
|
|
|
125
136
|
def last_error
|
|
126
137
|
@last_error[Thread.current.object_id]
|
|
@@ -130,7 +141,7 @@ module Fluent
|
|
|
130
141
|
@logger.debug { "event: #{tag} #{map.to_json}" rescue nil } if @logger.debug?
|
|
131
142
|
tag = "#{@tag_prefix}.#{tag}" if @tag_prefix
|
|
132
143
|
if @nanosecond_precision && time.is_a?(Time)
|
|
133
|
-
write [tag, EventTime.new(time), map]
|
|
144
|
+
write [tag, EventTime.new(time.to_i, time.nsec), map]
|
|
134
145
|
else
|
|
135
146
|
write [tag, time.to_i, map]
|
|
136
147
|
end
|
|
@@ -162,7 +173,36 @@ module Fluent
|
|
|
162
173
|
if @socket_path
|
|
163
174
|
@con = UNIXSocket.new(@socket_path)
|
|
164
175
|
else
|
|
165
|
-
@con =
|
|
176
|
+
@con = Timeout.timeout(@timeout) do
|
|
177
|
+
TCPSocket.new(@host, @port)
|
|
178
|
+
end
|
|
179
|
+
if @tls_options
|
|
180
|
+
context = OpenSSL::SSL::SSLContext.new
|
|
181
|
+
if @tls_options[:insecure]
|
|
182
|
+
context.verify_mode = OpenSSL::SSL::VERIFY_NONE
|
|
183
|
+
else
|
|
184
|
+
context.set_params({})
|
|
185
|
+
context.verify_mode = OpenSSL::SSL::VERIFY_PEER
|
|
186
|
+
cert_store = OpenSSL::X509::Store.new
|
|
187
|
+
if @tls_options[:use_default_ca]
|
|
188
|
+
cert_store.set_default_paths
|
|
189
|
+
end
|
|
190
|
+
if @tls_options[:ca]
|
|
191
|
+
cert_store.add_file(@tls_options[:ca])
|
|
192
|
+
end
|
|
193
|
+
|
|
194
|
+
context.cert = OpenSSL::X509::Certificate.new(File.read(@tls_options[:cert])) if @tls_options[:cert]
|
|
195
|
+
context.key = OpenSSL::PKey::read(File.read(@tls_options[:key]), @tls_options[:key_passphrase]) if @tls_options[:key]
|
|
196
|
+
context.ciphers = @tls_options[:ciphers] || "ALL:!aNULL:!eNULL:!SSLv2".freeze
|
|
197
|
+
context.cert_store = cert_store
|
|
198
|
+
end
|
|
199
|
+
set_tls_version(context)
|
|
200
|
+
|
|
201
|
+
@con = OpenSSL::SSL::SSLSocket.new(@con, context)
|
|
202
|
+
@con.sync_close = true
|
|
203
|
+
@con.connect
|
|
204
|
+
end
|
|
205
|
+
@con
|
|
166
206
|
end
|
|
167
207
|
end
|
|
168
208
|
|
|
@@ -180,14 +220,25 @@ module Fluent
|
|
|
180
220
|
|
|
181
221
|
private
|
|
182
222
|
|
|
223
|
+
def set_tls_version(context)
|
|
224
|
+
if context.respond_to?(:min_version=)
|
|
225
|
+
ver = @tls_options[:version] || OpenSSL::SSL::TLS1_2_VERSION
|
|
226
|
+
context.min_version = ver
|
|
227
|
+
context.max_version = ver
|
|
228
|
+
else
|
|
229
|
+
context.ssl_version = @tls_options[:version] || :'TLSv1_2'
|
|
230
|
+
end
|
|
231
|
+
end
|
|
232
|
+
|
|
183
233
|
def to_msgpack(msg)
|
|
184
234
|
@mon.synchronize {
|
|
185
235
|
res = begin
|
|
186
236
|
@packer.pack(msg).to_s
|
|
187
237
|
rescue NoMethodError
|
|
188
238
|
JSON.parse(JSON.generate(msg)).to_msgpack
|
|
239
|
+
ensure
|
|
240
|
+
@packer.clear
|
|
189
241
|
end
|
|
190
|
-
@packer.clear
|
|
191
242
|
res
|
|
192
243
|
}
|
|
193
244
|
end
|
|
@@ -224,10 +275,17 @@ module Fluent
|
|
|
224
275
|
end
|
|
225
276
|
|
|
226
277
|
begin
|
|
227
|
-
send_data(@pending)
|
|
278
|
+
written = send_data(@pending)
|
|
279
|
+
if @pending.bytesize != written
|
|
280
|
+
raise "Actual written data size(#{written} bytes) is different from the received data size(#{@pending.bytesize} bytes)."
|
|
281
|
+
end
|
|
282
|
+
|
|
228
283
|
@pending = nil
|
|
229
284
|
true
|
|
230
285
|
rescue => e
|
|
286
|
+
unless wait_writeable?(e)
|
|
287
|
+
raise e
|
|
288
|
+
end
|
|
231
289
|
set_last_error(e)
|
|
232
290
|
if pending_bytesize > @limit
|
|
233
291
|
@logger.error("FluentLogger: Can't send logs to #{connection_string}: #{$!}")
|
|
@@ -245,7 +303,15 @@ module Fluent
|
|
|
245
303
|
unless connect?
|
|
246
304
|
connect!
|
|
247
305
|
end
|
|
248
|
-
@
|
|
306
|
+
if @use_nonblock
|
|
307
|
+
send_data_nonblock(data)
|
|
308
|
+
else
|
|
309
|
+
_, ws = IO.select([], [@con])
|
|
310
|
+
Thread.handle_interrupt(::Timeout::Error => :never) do
|
|
311
|
+
# block timeout error during IO#write
|
|
312
|
+
ws.first.write(data)
|
|
313
|
+
end
|
|
314
|
+
end
|
|
249
315
|
#while true
|
|
250
316
|
# puts "sending #{data.length} bytes"
|
|
251
317
|
# if data.length > 32*1024
|
|
@@ -259,7 +325,19 @@ module Fluent
|
|
|
259
325
|
# end
|
|
260
326
|
# data = data[n..-1]
|
|
261
327
|
#end
|
|
262
|
-
|
|
328
|
+
end
|
|
329
|
+
|
|
330
|
+
def send_data_nonblock(data)
|
|
331
|
+
written = @con.write_nonblock(data)
|
|
332
|
+
remaining = data.bytesize - written
|
|
333
|
+
|
|
334
|
+
while remaining > 0
|
|
335
|
+
len = @con.write_nonblock(data.byteslice(written, remaining))
|
|
336
|
+
remaining -= len
|
|
337
|
+
written += len
|
|
338
|
+
end
|
|
339
|
+
|
|
340
|
+
written
|
|
263
341
|
end
|
|
264
342
|
|
|
265
343
|
def connect!
|
|
@@ -297,6 +375,14 @@ module Fluent
|
|
|
297
375
|
# TODO: Check non GVL env
|
|
298
376
|
@last_error[Thread.current.object_id] = e
|
|
299
377
|
end
|
|
378
|
+
|
|
379
|
+
def wait_writeable?(e)
|
|
380
|
+
if e.instance_of?(IO::EAGAINWaitWritable)
|
|
381
|
+
@wait_writeable
|
|
382
|
+
else
|
|
383
|
+
true
|
|
384
|
+
end
|
|
385
|
+
end
|
|
300
386
|
end
|
|
301
387
|
end
|
|
302
388
|
end
|
data/spec/fluent_logger_spec.rb
CHANGED
|
@@ -6,6 +6,7 @@ require 'support/dummy_fluentd'
|
|
|
6
6
|
require 'logger'
|
|
7
7
|
require 'stringio'
|
|
8
8
|
require 'fluent/logger/fluent_logger/cui'
|
|
9
|
+
require 'timeout'
|
|
9
10
|
|
|
10
11
|
describe Fluent::Logger::FluentLogger do
|
|
11
12
|
let(:fluentd) {
|
|
@@ -58,6 +59,57 @@ describe Fluent::Logger::FluentLogger do
|
|
|
58
59
|
@serverengine.shutdown
|
|
59
60
|
end
|
|
60
61
|
|
|
62
|
+
describe('testing interaction of use_nonblock and wait_writeable') do
|
|
63
|
+
before(:example) do
|
|
64
|
+
allow_any_instance_of(TCPSocket).to receive(:write_nonblock).and_raise(IO::EAGAINWaitWritable)
|
|
65
|
+
allow_any_instance_of(TCPSocket).to receive(:write) { |_, buf| buf.size }
|
|
66
|
+
end
|
|
67
|
+
|
|
68
|
+
context('use_nonblock is false') do
|
|
69
|
+
let(:block_config) { logger_config.merge(use_nonblock: false) }
|
|
70
|
+
|
|
71
|
+
it('post returns true when wait_writeable is false') {
|
|
72
|
+
cfg = block_config.merge(wait_writeable: false)
|
|
73
|
+
l = Fluent::Logger::FluentLogger.new('logger-test', cfg)
|
|
74
|
+
expect(l.post('hello', foo: 'bar')).to eq true
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
it('post returns true when wait_writeable is true') {
|
|
78
|
+
cfg = block_config.merge(wait_writeable: true)
|
|
79
|
+
l = Fluent::Logger::FluentLogger.new('logger-test', cfg)
|
|
80
|
+
expect(l.post('hello', {foo: 'bar'})).to eq true
|
|
81
|
+
}
|
|
82
|
+
end
|
|
83
|
+
|
|
84
|
+
context('use_nonblock is true') do
|
|
85
|
+
let(:nonblock_config) { logger_config.merge(use_nonblock: true) }
|
|
86
|
+
|
|
87
|
+
it('post raises IO::EAGAINWaitWritable when wait_writeable is false') {
|
|
88
|
+
cfg = nonblock_config.merge(wait_writeable: false)
|
|
89
|
+
l = Fluent::Logger::FluentLogger.new('logger-test', cfg)
|
|
90
|
+
expect { l.post('hello', foo: 'bar') }.to raise_error(IO::EAGAINWaitWritable)
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
it('post returns false when wait_writeable is true') {
|
|
94
|
+
cfg = nonblock_config.merge(wait_writeable: true)
|
|
95
|
+
l = Fluent::Logger::FluentLogger.new('logger-test', cfg)
|
|
96
|
+
expect(l.post('hello', {foo: 'bar'})).to eq false
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
context 'when write_nonblock returns the size less than received data' do
|
|
100
|
+
before do
|
|
101
|
+
allow_any_instance_of(TCPSocket).to receive(:write_nonblock).and_return(1) # write 1 bytes per call
|
|
102
|
+
end
|
|
103
|
+
|
|
104
|
+
it 'buffering data and flush at closed time' do
|
|
105
|
+
logger = Fluent::Logger::FluentLogger.new('logger-test', nonblock_config)
|
|
106
|
+
expect(logger.post('hello', foo: 'bar')).to eq(true)
|
|
107
|
+
expect(logger.pending_bytesize).to eq(0)
|
|
108
|
+
end
|
|
109
|
+
end
|
|
110
|
+
end
|
|
111
|
+
end
|
|
112
|
+
|
|
61
113
|
context('Post by CUI') do
|
|
62
114
|
it('post') {
|
|
63
115
|
args = %W(-h localhost -p #{fluentd.port} -t logger-test.tag -v a=b -v foo=bar)
|
|
@@ -86,6 +138,16 @@ describe Fluent::Logger::FluentLogger do
|
|
|
86
138
|
}
|
|
87
139
|
end
|
|
88
140
|
|
|
141
|
+
context 'when the message has object which does not have #to_msgpack method' do
|
|
142
|
+
it 'success with nanosecond' do
|
|
143
|
+
expect(logger_with_nanosec.pending_bytesize).to eq(0)
|
|
144
|
+
expect(logger_with_nanosec.post('tag', 'a' => Errno::ETIMEDOUT)).to eq(true)
|
|
145
|
+
fluentd.wait_transfer
|
|
146
|
+
expect(fluentd.queue.last).to eq(['logger-test.tag', { 'a' => 'Errno::ETIMEDOUT' }])
|
|
147
|
+
expect(logger_with_nanosec.pending_bytesize).to eq(0)
|
|
148
|
+
end
|
|
149
|
+
end
|
|
150
|
+
|
|
89
151
|
it ('close after post') {
|
|
90
152
|
expect(logger).to be_connect
|
|
91
153
|
logger.close
|
|
@@ -144,6 +206,10 @@ describe Fluent::Logger::FluentLogger do
|
|
|
144
206
|
expect(fluentd.queue.last).to be_nil
|
|
145
207
|
logger_io.rewind
|
|
146
208
|
logger_io.read =~ /FluentLogger: Can't convert to msgpack:/
|
|
209
|
+
|
|
210
|
+
logger.post('tag', { 'a' => 'b' })
|
|
211
|
+
fluentd.wait_transfer
|
|
212
|
+
expect(fluentd.queue.last).to eq ['logger-test.tag', { 'a' => 'b' }]
|
|
147
213
|
}
|
|
148
214
|
|
|
149
215
|
it ('should raise an error when second argument is non hash object') {
|
|
@@ -312,4 +378,70 @@ describe Fluent::Logger::FluentLogger do
|
|
|
312
378
|
end
|
|
313
379
|
end
|
|
314
380
|
end
|
|
381
|
+
|
|
382
|
+
context "running fluentd with TLS" do
|
|
383
|
+
before(:all) do
|
|
384
|
+
@serverengine = DummyServerengine.new
|
|
385
|
+
@serverengine.startup
|
|
386
|
+
end
|
|
387
|
+
|
|
388
|
+
before(:each) do
|
|
389
|
+
fluentd.startup(true)
|
|
390
|
+
# Make sure to wait TLS server since preparing it takes longer time than
|
|
391
|
+
# non-TLS so that it might be too early especially on CI environment
|
|
392
|
+
3.times do
|
|
393
|
+
begin
|
|
394
|
+
TCPSocket.open('localhost', fluentd.port) {}
|
|
395
|
+
break
|
|
396
|
+
rescue
|
|
397
|
+
fluentd.wait_transfer
|
|
398
|
+
end
|
|
399
|
+
end
|
|
400
|
+
end
|
|
401
|
+
|
|
402
|
+
after(:each) do
|
|
403
|
+
fluentd.shutdown
|
|
404
|
+
end
|
|
405
|
+
|
|
406
|
+
after(:all) do
|
|
407
|
+
@serverengine.shutdown
|
|
408
|
+
end
|
|
409
|
+
|
|
410
|
+
let(:logger_config) {
|
|
411
|
+
{
|
|
412
|
+
:host => 'localhost',
|
|
413
|
+
:port => fluentd.port,
|
|
414
|
+
:logger => internal_logger,
|
|
415
|
+
:buffer_overflow_handler => buffer_overflow_handler,
|
|
416
|
+
:tls_options => {:insecure => true}
|
|
417
|
+
}
|
|
418
|
+
}
|
|
419
|
+
|
|
420
|
+
context('post') do
|
|
421
|
+
it ('success') {
|
|
422
|
+
expect(logger.pending_bytesize).to eq 0
|
|
423
|
+
expect(logger.post('tag', {'a' => 'b'})).to be true
|
|
424
|
+
fluentd.wait_transfer
|
|
425
|
+
expect(fluentd.queue.last).to eq ['logger-test.tag', {'a' => 'b'}]
|
|
426
|
+
expect(logger.pending_bytesize).to eq 0
|
|
427
|
+
}
|
|
428
|
+
|
|
429
|
+
it ('success with nanosecond') {
|
|
430
|
+
expect(logger_with_nanosec.pending_bytesize).to eq 0
|
|
431
|
+
expect(logger_with_nanosec.post('tag', {'a' => 'b'})).to be true
|
|
432
|
+
fluentd.wait_transfer
|
|
433
|
+
expect(fluentd.queue.last).to eq ['logger-test.tag', {'a' => 'b'}]
|
|
434
|
+
expect(fluentd.output.emits.first[1]).to be_a_kind_of(Fluent::EventTime)
|
|
435
|
+
}
|
|
436
|
+
end
|
|
437
|
+
end
|
|
438
|
+
|
|
439
|
+
it ('support timeout') {
|
|
440
|
+
Timeout::timeout(5) do
|
|
441
|
+
# Use invalid IP address to make sure that the connection will timeout.
|
|
442
|
+
# (192.0.2.0 is a special IP address that can be used in only documentation. Ref. RFC 5737)
|
|
443
|
+
logger = Fluent::Logger::FluentLogger.new(nil, host: '192.0.2.0', port: fluentd.port, timeout: 1)
|
|
444
|
+
expect(logger.last_error).to be_a_kind_of(Timeout::Error)
|
|
445
|
+
end
|
|
446
|
+
}
|
|
315
447
|
end
|
|
@@ -51,8 +51,22 @@ class DummyFluentd
|
|
|
51
51
|
queue
|
|
52
52
|
end
|
|
53
53
|
|
|
54
|
-
def startup
|
|
55
|
-
|
|
54
|
+
def startup(with_tls = false)
|
|
55
|
+
if with_tls
|
|
56
|
+
config = Fluent::Config.parse(<<EOF, '(logger-spec)', '(logger-spec-dir)', true)
|
|
57
|
+
<source>
|
|
58
|
+
type forward
|
|
59
|
+
port #{port}
|
|
60
|
+
<transport tls>
|
|
61
|
+
insecure true
|
|
62
|
+
</transport>
|
|
63
|
+
</source>
|
|
64
|
+
<match logger-test.**>
|
|
65
|
+
type test
|
|
66
|
+
</match>
|
|
67
|
+
EOF
|
|
68
|
+
else
|
|
69
|
+
config = Fluent::Config.parse(<<EOF, '(logger-spec)', '(logger-spec-dir)', true)
|
|
56
70
|
<source>
|
|
57
71
|
type forward
|
|
58
72
|
port #{port}
|
|
@@ -61,6 +75,8 @@ class DummyFluentd
|
|
|
61
75
|
type test
|
|
62
76
|
</match>
|
|
63
77
|
EOF
|
|
78
|
+
end
|
|
79
|
+
|
|
64
80
|
Fluent::Test.setup
|
|
65
81
|
Fluent::Engine.run_configure(config)
|
|
66
82
|
@coolio_default_loop = nil
|
metadata
CHANGED
|
@@ -1,14 +1,13 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: fluent-logger
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.
|
|
4
|
+
version: 0.10.0
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Sadayuki Furuhashi
|
|
8
|
-
autorequire:
|
|
9
8
|
bindir: bin
|
|
10
9
|
cert_chain: []
|
|
11
|
-
date:
|
|
10
|
+
date: 1980-01-02 00:00:00.000000000 Z
|
|
12
11
|
dependencies:
|
|
13
12
|
- !ruby/object:Gem::Dependency
|
|
14
13
|
name: msgpack
|
|
@@ -31,75 +30,19 @@ dependencies:
|
|
|
31
30
|
- !ruby/object:Gem::Version
|
|
32
31
|
version: '2'
|
|
33
32
|
- !ruby/object:Gem::Dependency
|
|
34
|
-
name:
|
|
33
|
+
name: logger
|
|
35
34
|
requirement: !ruby/object:Gem::Requirement
|
|
36
35
|
requirements:
|
|
37
|
-
- - "
|
|
38
|
-
- !ruby/object:Gem::Version
|
|
39
|
-
version: 0.9.2
|
|
40
|
-
type: :development
|
|
41
|
-
prerelease: false
|
|
42
|
-
version_requirements: !ruby/object:Gem::Requirement
|
|
43
|
-
requirements:
|
|
44
|
-
- - ">="
|
|
45
|
-
- !ruby/object:Gem::Version
|
|
46
|
-
version: 0.9.2
|
|
47
|
-
- !ruby/object:Gem::Dependency
|
|
48
|
-
name: rspec
|
|
49
|
-
requirement: !ruby/object:Gem::Requirement
|
|
50
|
-
requirements:
|
|
51
|
-
- - ">="
|
|
36
|
+
- - "~>"
|
|
52
37
|
- !ruby/object:Gem::Version
|
|
53
|
-
version:
|
|
54
|
-
type: :
|
|
55
|
-
prerelease: false
|
|
56
|
-
version_requirements: !ruby/object:Gem::Requirement
|
|
57
|
-
requirements:
|
|
58
|
-
- - ">="
|
|
59
|
-
- !ruby/object:Gem::Version
|
|
60
|
-
version: 3.0.0
|
|
61
|
-
- !ruby/object:Gem::Dependency
|
|
62
|
-
name: rspec-its
|
|
63
|
-
requirement: !ruby/object:Gem::Requirement
|
|
64
|
-
requirements:
|
|
65
|
-
- - ">="
|
|
66
|
-
- !ruby/object:Gem::Version
|
|
67
|
-
version: 1.1.0
|
|
68
|
-
type: :development
|
|
69
|
-
prerelease: false
|
|
70
|
-
version_requirements: !ruby/object:Gem::Requirement
|
|
71
|
-
requirements:
|
|
72
|
-
- - ">="
|
|
73
|
-
- !ruby/object:Gem::Version
|
|
74
|
-
version: 1.1.0
|
|
75
|
-
- !ruby/object:Gem::Dependency
|
|
76
|
-
name: simplecov
|
|
77
|
-
requirement: !ruby/object:Gem::Requirement
|
|
78
|
-
requirements:
|
|
79
|
-
- - ">="
|
|
80
|
-
- !ruby/object:Gem::Version
|
|
81
|
-
version: 0.5.4
|
|
82
|
-
type: :development
|
|
83
|
-
prerelease: false
|
|
84
|
-
version_requirements: !ruby/object:Gem::Requirement
|
|
85
|
-
requirements:
|
|
86
|
-
- - ">="
|
|
87
|
-
- !ruby/object:Gem::Version
|
|
88
|
-
version: 0.5.4
|
|
89
|
-
- !ruby/object:Gem::Dependency
|
|
90
|
-
name: timecop
|
|
91
|
-
requirement: !ruby/object:Gem::Requirement
|
|
92
|
-
requirements:
|
|
93
|
-
- - ">="
|
|
94
|
-
- !ruby/object:Gem::Version
|
|
95
|
-
version: 0.3.0
|
|
96
|
-
type: :development
|
|
38
|
+
version: '1.6'
|
|
39
|
+
type: :runtime
|
|
97
40
|
prerelease: false
|
|
98
41
|
version_requirements: !ruby/object:Gem::Requirement
|
|
99
42
|
requirements:
|
|
100
|
-
- - "
|
|
43
|
+
- - "~>"
|
|
101
44
|
- !ruby/object:Gem::Version
|
|
102
|
-
version:
|
|
45
|
+
version: '1.6'
|
|
103
46
|
description: fluent logger for ruby
|
|
104
47
|
email: frsyuki@gmail.com
|
|
105
48
|
executables:
|
|
@@ -107,9 +50,9 @@ executables:
|
|
|
107
50
|
extensions: []
|
|
108
51
|
extra_rdoc_files: []
|
|
109
52
|
files:
|
|
53
|
+
- ".github/workflows/linux.yml"
|
|
110
54
|
- ".gitignore"
|
|
111
55
|
- ".rspec"
|
|
112
|
-
- ".travis.yml"
|
|
113
56
|
- AUTHORS
|
|
114
57
|
- COPYING
|
|
115
58
|
- ChangeLog
|
|
@@ -117,7 +60,6 @@ files:
|
|
|
117
60
|
- Gemfile.v0.12
|
|
118
61
|
- README.md
|
|
119
62
|
- Rakefile
|
|
120
|
-
- VERSION
|
|
121
63
|
- bin/fluent-post
|
|
122
64
|
- fluent-logger.gemspec
|
|
123
65
|
- lib/fluent-logger.rb
|
|
@@ -144,9 +86,9 @@ files:
|
|
|
144
86
|
- spec/support/timecop.rb
|
|
145
87
|
- spec/test_logger_spec.rb
|
|
146
88
|
homepage: https://github.com/fluent/fluent-logger-ruby
|
|
147
|
-
licenses:
|
|
89
|
+
licenses:
|
|
90
|
+
- Apache-2.0
|
|
148
91
|
metadata: {}
|
|
149
|
-
post_install_message:
|
|
150
92
|
rdoc_options: []
|
|
151
93
|
require_paths:
|
|
152
94
|
- lib
|
|
@@ -161,9 +103,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
|
161
103
|
- !ruby/object:Gem::Version
|
|
162
104
|
version: '0'
|
|
163
105
|
requirements: []
|
|
164
|
-
|
|
165
|
-
rubygems_version: 2.6.11
|
|
166
|
-
signing_key:
|
|
106
|
+
rubygems_version: 3.6.9
|
|
167
107
|
specification_version: 4
|
|
168
108
|
summary: fluent logger for ruby
|
|
169
109
|
test_files:
|
data/.travis.yml
DELETED
|
@@ -1,34 +0,0 @@
|
|
|
1
|
-
rvm:
|
|
2
|
-
- 1.9.3
|
|
3
|
-
- 2.0.0
|
|
4
|
-
- 2.1
|
|
5
|
-
- 2.2
|
|
6
|
-
- 2.3.1
|
|
7
|
-
- 2.4.0
|
|
8
|
-
- ruby-head
|
|
9
|
-
|
|
10
|
-
gemfile:
|
|
11
|
-
- Gemfile
|
|
12
|
-
- Gemfile.v0.12
|
|
13
|
-
|
|
14
|
-
before_install: gem update bundler
|
|
15
|
-
script: bundle exec rake spec
|
|
16
|
-
|
|
17
|
-
sudo: false
|
|
18
|
-
|
|
19
|
-
matrix:
|
|
20
|
-
allow_failures:
|
|
21
|
-
- rvm: ruby-head
|
|
22
|
-
exclude:
|
|
23
|
-
- rvm: 1.9.3
|
|
24
|
-
gemfile: Gemfile
|
|
25
|
-
- rvm: 2.0.0
|
|
26
|
-
gemfile: Gemfile
|
|
27
|
-
- rvm: 2.2
|
|
28
|
-
gemfile: Gemfile.v0.12
|
|
29
|
-
- rvm: 2.3.1
|
|
30
|
-
gemfile: Gemfile.v0.12
|
|
31
|
-
- rvm: 2.4.0
|
|
32
|
-
gemfile: Gemfile.v0.12
|
|
33
|
-
- rvm: ruby-head
|
|
34
|
-
gemfile: Gemfile.v0.12
|
data/VERSION
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
0.7.1
|