fluent-plugin-scalyr 0.8.14 → 0.8.17
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/VERSION +1 -1
- data/fluent-plugin-scalyr.gemspec +1 -1
- data/lib/fluent/plugin/out_scalyr.rb +42 -8
- data/test/test_events.rb +44 -0
- metadata +5 -5
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: df57f8a053be1230c4af77591eb3b2cab52dd23ce2733a38046a6d03a1ab24cb
|
4
|
+
data.tar.gz: 20675ccb717aa4c71f216c619070d78dcde0123a43f1866aa43387f84e4a856d
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 7036556c268716b93ca61a27b02a2bfd7d1760b6eaecb15423491113c0ba6e97a0472df6d69776fc41b0bb3266e2554944e8853a788020637ccc781d320b786e
|
7
|
+
data.tar.gz: 767f067ee3f88c0af7abd58049f11afd3305f7c0a8ae6885e2a6cca5f57318b1b111869fc3fb11a2e6f61bcd627abdf9e6a47435fab5733cdee826bd1493a741
|
data/VERSION
CHANGED
@@ -1 +1 @@
|
|
1
|
-
0.8.
|
1
|
+
0.8.17
|
@@ -23,7 +23,7 @@ Gem::Specification.new do |gem|
|
|
23
23
|
gem.add_dependency "zlib"
|
24
24
|
gem.add_development_dependency "bundler", "~> 1.9"
|
25
25
|
gem.add_development_dependency "flexmock", "~> 1.2"
|
26
|
-
gem.add_development_dependency "rake", "~> 0
|
26
|
+
gem.add_development_dependency "rake", "~> 13.0"
|
27
27
|
gem.add_development_dependency "rubocop", "~> 0.4"
|
28
28
|
gem.add_development_dependency "test-unit", "~> 3.0"
|
29
29
|
end
|
@@ -73,6 +73,12 @@ module Scalyr
|
|
73
73
|
end
|
74
74
|
|
75
75
|
def configure(conf)
|
76
|
+
@version = if Gem.loaded_specs.key?("fluent-plugin-scalyr")
|
77
|
+
Gem.loaded_specs["fluent-plugin-scalyr"].version
|
78
|
+
else
|
79
|
+
"unknown"
|
80
|
+
end
|
81
|
+
|
76
82
|
if conf.elements("buffer").empty?
|
77
83
|
$log.warn "Pre 0.14.0 configuration file detected. Please consider updating your configuration file" # rubocop:disable Layout/LineLength, Lint/RedundantCopDisableDirective
|
78
84
|
end
|
@@ -146,7 +152,7 @@ module Scalyr
|
|
146
152
|
# Generate a session id. This will be called once for each <match> in fluent.conf that uses scalyr
|
147
153
|
@session = SecureRandom.uuid
|
148
154
|
|
149
|
-
$log.info "Scalyr Fluentd Plugin ID id=#{plugin_id} worker=#{fluentd_worker_id} session=#{@session}" # rubocop:disable Layout/LineLength, Lint/RedundantCopDisableDirective
|
155
|
+
$log.info "Scalyr Fluentd Plugin ID id=#{plugin_id} worker=#{fluentd_worker_id} session=#{@session} version=#{@version}" # rubocop:disable Layout/LineLength, Lint/RedundantCopDisableDirective
|
150
156
|
end
|
151
157
|
|
152
158
|
def format(tag, time, record)
|
@@ -254,8 +260,8 @@ module Scalyr
|
|
254
260
|
|
255
261
|
post = Net::HTTP::Post.new uri.path
|
256
262
|
post.add_field("Content-Type", "application/json")
|
257
|
-
|
258
263
|
post.add_field("Content-Encoding", encoding) if @compression_type
|
264
|
+
post.add_field("User-Agent", "fluent-plugin-scalyr;#{@version}")
|
259
265
|
|
260
266
|
post.body = body
|
261
267
|
|
@@ -358,13 +364,37 @@ module Scalyr
|
|
358
364
|
# generate new request if json size of events in the array exceed maximum request buffer size
|
359
365
|
append_event = true
|
360
366
|
if total_bytes + event_json.bytesize > @max_request_buffer
|
361
|
-
#
|
367
|
+
# the case where a single event causes us to exceed the @max_request_buffer
|
362
368
|
if events.empty?
|
363
|
-
|
369
|
+
# if we are able to truncate the content (and append an ellipsis)
|
370
|
+
# inside the @message_field we do so here
|
371
|
+
if record.key?(@message_field) &&
|
372
|
+
record[@message_field].is_a?(String) &&
|
373
|
+
record[@message_field].bytesize > event_json.bytesize - @max_request_buffer &&
|
374
|
+
record[@message_field].bytesize >= 3
|
375
|
+
|
376
|
+
@log.warn "Received a record that cannot fit within max_request_buffer "\
|
377
|
+
"(#{@max_request_buffer}) from #{record['logfile']}, serialized event size "\
|
378
|
+
"is #{event_json.bytesize}. The #{@message_field} field will be truncated to fit."
|
379
|
+
max_msg_size = @max_request_buffer - event_json.bytesize - 3
|
380
|
+
truncated_msg = event[:attrs][@message_field][0...max_msg_size] + "..."
|
381
|
+
event[:attrs][@message_field] = truncated_msg
|
382
|
+
events << event
|
383
|
+
|
384
|
+
# otherwise we drop the event and save ourselves hitting a 4XX response from the server
|
385
|
+
else
|
386
|
+
@log.warn "Received a record that cannot fit within max_request_buffer "\
|
387
|
+
"(#{@max_request_buffer}) from #{record['logfile']}, serialized event size "\
|
388
|
+
"is #{event_json.bytesize}. The #{@message_field} field too short to truncate, "\
|
389
|
+
"dropping event."
|
390
|
+
end
|
364
391
|
append_event = false
|
365
392
|
end
|
366
|
-
|
367
|
-
|
393
|
+
|
394
|
+
unless events.empty?
|
395
|
+
request = create_request(events, current_threads)
|
396
|
+
requests << request
|
397
|
+
end
|
368
398
|
|
369
399
|
total_bytes = 0
|
370
400
|
current_threads = {}
|
@@ -380,8 +410,12 @@ module Scalyr
|
|
380
410
|
}
|
381
411
|
|
382
412
|
# create a final request with any left over events
|
383
|
-
|
384
|
-
|
413
|
+
unless events.empty?
|
414
|
+
request = create_request(events, current_threads)
|
415
|
+
requests << request
|
416
|
+
end
|
417
|
+
|
418
|
+
requests
|
385
419
|
end
|
386
420
|
|
387
421
|
def create_request(events, current_threads)
|
data/test/test_events.rb
CHANGED
@@ -357,4 +357,48 @@ class EventsTest < Scalyr::ScalyrOutTest
|
|
357
357
|
d.feed(time, attrs)
|
358
358
|
end
|
359
359
|
end
|
360
|
+
|
361
|
+
def test_truncated_large_event
|
362
|
+
d = create_driver CONFIG + "max_request_buffer 4000"
|
363
|
+
|
364
|
+
time = event_time("2015-04-01 10:00:00 UTC")
|
365
|
+
attrs = {"log" => "this is a test", "message" => "0123456789" * 500}
|
366
|
+
|
367
|
+
response = flexmock(Net::HTTPResponse, code: "200", body: '{ "status":"success" }')
|
368
|
+
mock = flexmock(d.instance)
|
369
|
+
|
370
|
+
mock.should_receive(:post_request).with(
|
371
|
+
URI,
|
372
|
+
on {|request_body|
|
373
|
+
body = JSON.parse(request_body)
|
374
|
+
events = body["events"]
|
375
|
+
assert(events[0]["attrs"].key?("message"), "'message' field not found in event")
|
376
|
+
assert_equal(
|
377
|
+
"0123456789" * 388 + "012...",
|
378
|
+
events[0]["attrs"]["message"],
|
379
|
+
"'message' field incorrect"
|
380
|
+
)
|
381
|
+
true
|
382
|
+
}
|
383
|
+
).once.and_return(response)
|
384
|
+
|
385
|
+
d.run(default_tag: "test") do
|
386
|
+
d.feed(time, attrs)
|
387
|
+
end
|
388
|
+
end
|
389
|
+
|
390
|
+
def test_dropped_large_event
|
391
|
+
d = create_driver CONFIG + "max_request_buffer 4000"
|
392
|
+
|
393
|
+
time = event_time("2015-04-01 10:00:00 UTC")
|
394
|
+
attrs = {"message" => "this is a test", "not_message" => "0123456789" * 500}
|
395
|
+
|
396
|
+
mock = flexmock(d.instance)
|
397
|
+
|
398
|
+
mock.should_receive(:post_request).never
|
399
|
+
|
400
|
+
d.run(default_tag: "test") do
|
401
|
+
d.feed(time, attrs)
|
402
|
+
end
|
403
|
+
end
|
360
404
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-scalyr
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.8.
|
4
|
+
version: 0.8.17
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Imron Alston
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2022-07-18 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: ffi
|
@@ -106,14 +106,14 @@ dependencies:
|
|
106
106
|
requirements:
|
107
107
|
- - "~>"
|
108
108
|
- !ruby/object:Gem::Version
|
109
|
-
version: '0
|
109
|
+
version: '13.0'
|
110
110
|
type: :development
|
111
111
|
prerelease: false
|
112
112
|
version_requirements: !ruby/object:Gem::Requirement
|
113
113
|
requirements:
|
114
114
|
- - "~>"
|
115
115
|
- !ruby/object:Gem::Version
|
116
|
-
version: '0
|
116
|
+
version: '13.0'
|
117
117
|
- !ruby/object:Gem::Dependency
|
118
118
|
name: rubocop
|
119
119
|
requirement: !ruby/object:Gem::Requirement
|
@@ -184,7 +184,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
184
184
|
- !ruby/object:Gem::Version
|
185
185
|
version: '0'
|
186
186
|
requirements: []
|
187
|
-
rubygems_version: 3.
|
187
|
+
rubygems_version: 3.3.11
|
188
188
|
signing_key:
|
189
189
|
specification_version: 4
|
190
190
|
summary: Scalyr plugin for fluentd
|