fluent-plugin-scalyr 0.7.4 → 0.8.3
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/README.md +24 -19
- data/VERSION +1 -1
- data/fluent-plugin-scalyr.gemspec +5 -5
- data/fluent.conf.sample +9 -8
- data/lib/fluent/plugin/out_scalyr.rb +66 -24
- data/test/helper.rb +1 -0
- metadata +7 -7
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: '081b15ca4f83f95b12bb784007ce3c77c2a572df'
|
4
|
+
data.tar.gz: 3194d197fed3ecda3fe0d8ff1a2e037d1292159e
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: e1b5ff7132f4ed915a8983b4bb7588895bed97a4b04accd77b1d8178e1e2ca9da1966e3eb4fc238a0573fa249584fa5ed83fecd6bb9fcc4c89b638fc09c6ee80
|
7
|
+
data.tar.gz: fa99e2c2fbde42d8df67103118479f5133f7bee64cb279a3665722cab373131ad12cffdab6287b9852b610279501cf1b8d74d2432fe465bb94d171dac2fc02c6
|
data/README.md
CHANGED
@@ -5,8 +5,11 @@ Scalyr output plugin for Fluentd
|
|
5
5
|
version 0.12 and 0.14.
|
6
6
|
|
7
7
|
The current master branch of the scalyr-fluentd plugin is compatible with
|
8
|
-
Fluentd version 0.14 and above
|
9
|
-
|
8
|
+
Fluentd version 0.14 and above, including fluentd v1.0.0.
|
9
|
+
|
10
|
+
If you wish to use the scalyr-fluentd plugin with Fluentd version 0.12 or
|
11
|
+
earlier, please use the [fluentd-0.12
|
12
|
+
branch](https://github.com/scalyr/scalyr-fluentd/tree/cbd8c2aac3d11472121345c7cad6587a7f68c115).
|
10
13
|
|
11
14
|
Overview
|
12
15
|
--------
|
@@ -36,7 +39,7 @@ For example, if you want to use Scalyr's ```accessLog``` parser for all events w
|
|
36
39
|
|
37
40
|
```
|
38
41
|
<filter scalyr.access>
|
39
|
-
type record_transformer
|
42
|
+
@type record_transformer
|
40
43
|
<record>
|
41
44
|
parser accessLog
|
42
45
|
</record>
|
@@ -50,7 +53,7 @@ The Scalyr output plugin has a number of sensible defaults so the minimum config
|
|
50
53
|
|
51
54
|
```
|
52
55
|
<match scalyr.*>
|
53
|
-
type
|
56
|
+
@type scalyr
|
54
57
|
api_write_token YOUR_SCALYR_WRITE_LOGS_TOKEN
|
55
58
|
</match>
|
56
59
|
```
|
@@ -59,7 +62,7 @@ The following configuration options are also supported:
|
|
59
62
|
|
60
63
|
```
|
61
64
|
<match scalyr.*>
|
62
|
-
type
|
65
|
+
@type scalyr
|
63
66
|
|
64
67
|
#scalyr specific options
|
65
68
|
api_write_token YOUR_SCALYR_WRITE_TOKEN
|
@@ -81,13 +84,15 @@ The following configuration options are also supported:
|
|
81
84
|
replace_invalid_utf8 false
|
82
85
|
|
83
86
|
#buffered output options
|
84
|
-
|
85
|
-
|
86
|
-
|
87
|
-
|
88
|
-
|
89
|
-
|
90
|
-
|
87
|
+
<buffer>
|
88
|
+
retry_max_times 40
|
89
|
+
retry_wait 5s
|
90
|
+
retry_max_interval 30s
|
91
|
+
flush_interval 5s
|
92
|
+
flush_thread_count 1
|
93
|
+
chunk_limit_size 100k
|
94
|
+
queue_limit_length 1024
|
95
|
+
</buffer>
|
91
96
|
|
92
97
|
</match>
|
93
98
|
```
|
@@ -124,21 +129,21 @@ The cURL project maintains CA certificate bundles automatically converted from m
|
|
124
129
|
|
125
130
|
***replace_invalid_utf8*** - If this value is true and ***force_message_encoding*** is set to 'UTF-8' then all invalid UTF-8 sequences in log messages will be replaced with <?>. Defaults to false. This flag has no effect if ***force_message_encoding*** is not set to 'UTF-8'.
|
126
131
|
|
127
|
-
####
|
132
|
+
####Buffer options
|
128
133
|
|
129
|
-
***
|
134
|
+
***retry_max_times*** - the maximum number of times to retry a failed post request before giving up. Defaults to *40*.
|
130
135
|
|
131
|
-
***retry_wait*** - the initial time to wait before retrying a failed request. Defaults to *5 seconds*. Wait times will increase up to a maximum of ***
|
136
|
+
***retry_wait*** - the initial time to wait before retrying a failed request. Defaults to *5 seconds*. Wait times will increase up to a maximum of ***retry_max_interval***
|
132
137
|
|
133
|
-
***
|
138
|
+
***retry_max_interval*** - the maximum time to wait between retrying failed requests. Defaults to *30 seconds*. **Note:** This is not the total maximum time of all retry waits, but rather the maximum time to wait for a single retry.
|
134
139
|
|
135
140
|
***flush_interval*** - how often to upload logs to Scalyr. Defaults to *5 seconds*.
|
136
141
|
|
137
|
-
***
|
142
|
+
***flush_thread_count*** - the number of threads to use to upload logs. This is currently fixed to 1 will cause fluentd to fail with a ConfigError if set to anything greater.
|
138
143
|
|
139
|
-
***
|
144
|
+
***chunk_limit_size*** - the maximum amount of log data to send to Scalyr in a single request. Defaults to *100KB*. **Note:** if you set this value too large, then Scalyr may reject your requests. Requests smaller than 1MB will typically be accepted by Scalyr, but note that the 1MB limit also includes the entire request body and all associated JSON keys and punctuation, which may be considerably larger than the raw log data.
|
140
145
|
|
141
|
-
***
|
146
|
+
***queue_limit_length*** - the maximum number of chunks to buffer before dropping new log requests. Defaults to *1024*. Combines with ***chunk_limit_size*** to give you the total amount of buffer to use in the event of request failures before dropping requests.
|
142
147
|
|
143
148
|
Secondary Logging
|
144
149
|
-----------------
|
data/VERSION
CHANGED
@@ -1 +1 @@
|
|
1
|
-
0.
|
1
|
+
0.8.3
|
@@ -8,14 +8,14 @@ Gem::Specification.new do |gem|
|
|
8
8
|
gem.version = File.read("VERSION").strip
|
9
9
|
gem.authors = ["Imron Alston"]
|
10
10
|
gem.licenses = ["Apache-2.0"]
|
11
|
-
gem.email = "imron@
|
11
|
+
gem.email = "imron@scalyr.com"
|
12
12
|
gem.has_rdoc = false
|
13
13
|
gem.platform = Gem::Platform::RUBY
|
14
|
-
gem.files =
|
15
|
-
gem.test_files =
|
16
|
-
gem.executables =
|
14
|
+
gem.files = Dir['AUTHORS', 'Gemfile', 'LICENSE', 'README.md', 'Rakefile', 'VERSION', 'fluent-plugin-scalyr.gemspec', 'fluent.conf.sample', 'lib/**/*', 'test/**/*']
|
15
|
+
gem.test_files = Dir.glob("{test,spec,features}/**/*")
|
16
|
+
gem.executables = Dir.glob("bin/*").map{ |f| File.basename(f) }
|
17
17
|
gem.require_paths = ['lib']
|
18
|
-
gem.add_dependency "fluentd", [">= 0.
|
18
|
+
gem.add_dependency "fluentd", [">= 0.14.0", "< 2"]
|
19
19
|
gem.add_development_dependency "rake", "~> 0.9"
|
20
20
|
gem.add_development_dependency "test-unit", "~> 3.0"
|
21
21
|
gem.add_development_dependency "flexmock", "~> 1.2"
|
data/fluent.conf.sample
CHANGED
@@ -16,13 +16,14 @@
|
|
16
16
|
# replace_invalid_utf8 false
|
17
17
|
|
18
18
|
|
19
|
-
|
20
|
-
#
|
21
|
-
#
|
22
|
-
#
|
23
|
-
#
|
24
|
-
#
|
25
|
-
#
|
26
|
-
#
|
19
|
+
<buffer>
|
20
|
+
# retry_max_times 40
|
21
|
+
# retry_wait 5s
|
22
|
+
# retry_max_interval 30s
|
23
|
+
# flush_interval 5s
|
24
|
+
# flush_thread_count 1
|
25
|
+
# chunk_limit_size 100k
|
26
|
+
# queue_limit_length 1024
|
27
|
+
</buffer>
|
27
28
|
|
28
29
|
</match>
|
@@ -16,7 +16,9 @@
|
|
16
16
|
# limitations under the License.
|
17
17
|
|
18
18
|
|
19
|
+
require 'fluent/plugin/output'
|
19
20
|
require 'fluent/plugin/scalyr-exceptions'
|
21
|
+
require 'fluent/plugin_helper/compat_parameters'
|
20
22
|
require 'json'
|
21
23
|
require 'net/http'
|
22
24
|
require 'net/https'
|
@@ -24,8 +26,9 @@ require 'securerandom'
|
|
24
26
|
require 'thread'
|
25
27
|
|
26
28
|
module Scalyr
|
27
|
-
class ScalyrOut < Fluent::
|
29
|
+
class ScalyrOut < Fluent::Plugin::Output
|
28
30
|
Fluent::Plugin.register_output( 'scalyr', self )
|
31
|
+
helpers :compat_parameters
|
29
32
|
|
30
33
|
config_param :api_write_token, :string
|
31
34
|
config_param :server_attributes, :hash, :default => nil
|
@@ -38,23 +41,35 @@ module Scalyr
|
|
38
41
|
config_param :force_message_encoding, :string, :default => nil
|
39
42
|
config_param :replace_invalid_utf8, :bool, :default => false
|
40
43
|
|
41
|
-
|
42
|
-
|
43
|
-
|
44
|
-
|
44
|
+
config_section :buffer do
|
45
|
+
config_set_default :retry_max_times, 40 #try a maximum of 40 times before discarding
|
46
|
+
config_set_default :retry_max_interval, 30 #wait a maximum of 30 seconds per retry
|
47
|
+
config_set_default :retry_wait, 5 #wait a minimum of 5 seconds per retry
|
48
|
+
config_set_default :flush_interval, 5 #default flush interval of 5 seconds
|
49
|
+
config_set_default :chunk_limit_size, 1024*100 #default chunk size of 100k
|
50
|
+
config_set_default :queue_limit_length, 1024 #default queue size of 1024
|
51
|
+
end
|
52
|
+
|
53
|
+
# support for version 0.14.0:
|
54
|
+
def compat_parameters_default_chunk_key
|
55
|
+
""
|
56
|
+
end
|
57
|
+
|
58
|
+
def formatted_to_msgpack_binary
|
59
|
+
true
|
60
|
+
end
|
45
61
|
|
46
62
|
def configure( conf )
|
47
|
-
|
48
|
-
|
49
|
-
|
50
|
-
conf["buffer_chunk_limit"] = "100k"
|
51
|
-
end
|
52
|
-
if !conf.key? "buffer_queue_limit"
|
53
|
-
conf["buffer_queue_limit"] = 1024
|
63
|
+
|
64
|
+
if conf.elements('buffer').empty?
|
65
|
+
$log.warn "Pre 0.14.0 configuration file detected. Please consider updating your configuration file"
|
54
66
|
end
|
67
|
+
|
68
|
+
compat_parameters_buffer( conf, default_chunk_key: '' )
|
69
|
+
|
55
70
|
super
|
56
71
|
|
57
|
-
if @buffer.
|
72
|
+
if @buffer.chunk_limit_size > 1024*1024
|
58
73
|
$log.warn "Buffer chunk size is greater than 1Mb. This may result in requests being rejected by Scalyr"
|
59
74
|
end
|
60
75
|
|
@@ -72,12 +87,30 @@ module Scalyr
|
|
72
87
|
end
|
73
88
|
end
|
74
89
|
|
90
|
+
#evaluate any statements in string value of the server_attributes object
|
91
|
+
if @server_attributes
|
92
|
+
new_attributes = {}
|
93
|
+
@server_attributes.each do |key, value|
|
94
|
+
if value.is_a?( String )
|
95
|
+
m = /^\#{(.*)}$/.match( value )
|
96
|
+
if m
|
97
|
+
new_attributes[key] = eval( m[1] )
|
98
|
+
else
|
99
|
+
new_attributes[key] = value
|
100
|
+
end
|
101
|
+
end
|
102
|
+
end
|
103
|
+
@server_attributes = new_attributes
|
104
|
+
end
|
105
|
+
|
75
106
|
@scalyr_server << '/' unless @scalyr_server.end_with?('/')
|
76
107
|
|
77
108
|
@add_events_uri = URI @scalyr_server + "addEvents"
|
78
109
|
|
110
|
+
num_threads = @buffer_config.flush_thread_count
|
111
|
+
|
79
112
|
#forcibly limit the number of threads to 1 for now, to ensure requests always have incrementing timestamps
|
80
|
-
raise Fluent::ConfigError, "num_threads is currently limited to 1. You specified #{
|
113
|
+
raise Fluent::ConfigError, "num_threads is currently limited to 1. You specified #{num_threads}." if num_threads > 1
|
81
114
|
end
|
82
115
|
|
83
116
|
def start
|
@@ -90,12 +123,21 @@ module Scalyr
|
|
90
123
|
#the following variables are all under the control of the above mutex
|
91
124
|
@thread_ids = Hash.new #hash of tags -> id
|
92
125
|
@next_id = 1 #incrementing thread id for the session
|
93
|
-
@last_timestamp = 0 #timestamp of most recent event
|
126
|
+
@last_timestamp = 0 #timestamp of most recent event in nanoseconds since epoch
|
94
127
|
|
95
128
|
end
|
96
129
|
|
97
130
|
def format( tag, time, record )
|
98
131
|
begin
|
132
|
+
|
133
|
+
if time.nil?
|
134
|
+
time = Fluent::Engine.now
|
135
|
+
end
|
136
|
+
|
137
|
+
if time.is_a?( Integer )
|
138
|
+
time = Fluent::EventTime.new( time )
|
139
|
+
end
|
140
|
+
|
99
141
|
if @message_field != "message"
|
100
142
|
if record.key? @message_field
|
101
143
|
if record.key? "message"
|
@@ -113,7 +155,7 @@ module Scalyr
|
|
113
155
|
record["message"].force_encoding( @message_encoding )
|
114
156
|
end
|
115
157
|
end
|
116
|
-
[tag, time, record].to_msgpack
|
158
|
+
[tag, time.sec, time.nsec, record].to_msgpack
|
117
159
|
|
118
160
|
rescue JSON::GeneratorError
|
119
161
|
$log.warn "Unable to format message due to JSON::GeneratorError. Record is:\n\t#{record.to_s}"
|
@@ -152,17 +194,16 @@ module Scalyr
|
|
152
194
|
end
|
153
195
|
|
154
196
|
|
155
|
-
|
156
197
|
#explicit function to convert to nanoseconds
|
157
198
|
#will make things easier to maintain if/when fluentd supports higher than second resolutions
|
158
|
-
def to_nanos( seconds )
|
159
|
-
seconds * 10**9
|
199
|
+
def to_nanos( seconds, nsec )
|
200
|
+
(seconds * 10**9) + nsec
|
160
201
|
end
|
161
202
|
|
162
203
|
#explicit function to convert to milliseconds
|
163
204
|
#will make things easier to maintain if/when fluentd supports higher than second resolutions
|
164
|
-
def to_millis(
|
165
|
-
|
205
|
+
def to_millis( timestamp )
|
206
|
+
(timestamp.sec * 10**3) + (timestamp.nsec / 10**6)
|
166
207
|
end
|
167
208
|
|
168
209
|
def post_request( uri, body )
|
@@ -238,9 +279,10 @@ module Scalyr
|
|
238
279
|
|
239
280
|
#create a Scalyr event object for each record in the chunk
|
240
281
|
events = Array.new
|
241
|
-
chunk.msgpack_each {|(tag,
|
282
|
+
chunk.msgpack_each {|(tag, sec, nsec, record)|
|
283
|
+
|
284
|
+
timestamp = self.to_nanos( sec, nsec )
|
242
285
|
|
243
|
-
timestamp = self.to_nanos( time )
|
244
286
|
thread_id = 0
|
245
287
|
|
246
288
|
@sync.synchronize {
|
@@ -268,7 +310,7 @@ module Scalyr
|
|
268
310
|
|
269
311
|
#append to list of events
|
270
312
|
event = { :thread => thread_id.to_s,
|
271
|
-
:ts => timestamp
|
313
|
+
:ts => timestamp,
|
272
314
|
:attrs => record
|
273
315
|
}
|
274
316
|
|
data/test/helper.rb
CHANGED
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-scalyr
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.8.3
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Imron Alston
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2018-02-
|
11
|
+
date: 2018-02-14 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: fluentd
|
@@ -16,20 +16,20 @@ dependencies:
|
|
16
16
|
requirements:
|
17
17
|
- - ">="
|
18
18
|
- !ruby/object:Gem::Version
|
19
|
-
version: 0.
|
19
|
+
version: 0.14.0
|
20
20
|
- - "<"
|
21
21
|
- !ruby/object:Gem::Version
|
22
|
-
version: '
|
22
|
+
version: '2'
|
23
23
|
type: :runtime
|
24
24
|
prerelease: false
|
25
25
|
version_requirements: !ruby/object:Gem::Requirement
|
26
26
|
requirements:
|
27
27
|
- - ">="
|
28
28
|
- !ruby/object:Gem::Version
|
29
|
-
version: 0.
|
29
|
+
version: 0.14.0
|
30
30
|
- - "<"
|
31
31
|
- !ruby/object:Gem::Version
|
32
|
-
version: '
|
32
|
+
version: '2'
|
33
33
|
- !ruby/object:Gem::Dependency
|
34
34
|
name: rake
|
35
35
|
requirement: !ruby/object:Gem::Requirement
|
@@ -87,7 +87,7 @@ dependencies:
|
|
87
87
|
- !ruby/object:Gem::Version
|
88
88
|
version: '1.9'
|
89
89
|
description: Sends log data collected by fluentd to Scalyr (http://www.scalyr.com)
|
90
|
-
email: imron@
|
90
|
+
email: imron@scalyr.com
|
91
91
|
executables: []
|
92
92
|
extensions: []
|
93
93
|
extra_rdoc_files: []
|