logstash-output-scalyr 0.1.23.beta → 0.1.24.beta
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/logstash/outputs/scalyr.rb +7 -3
- data/lib/scalyr/constants.rb +1 -1
- data/logstash-output-scalyr.gemspec +1 -1
- data/spec/logstash/outputs/scalyr_spec.rb +118 -0
- metadata +1 -1
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 312d501ddc98f27b8d2be32bb20cb29439e7ce096d5f12870ef1b1bf095c32ee
|
|
4
|
+
data.tar.gz: c0f6c4a51f42acef8266ee30480fc00a19e59098d7b957deddcc017bade71695
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 2628a0bcfbe9a13dfc5bb9fd93360ef7629fca158cc325184817b4f15492ff5fcb28c03ca992c08dbbf9dc7953d2715b071b5d94f67912c36f66625f7bcf3234
|
|
7
|
+
data.tar.gz: 1d009fe06794e8f2d15737da874609600e57569fe3d6be0297fb23a1ab99d375f3ac2e9bdbf52b653baa85222bc3d5da001ac8d68a3848c8099ed7eb1510175f
|
|
@@ -746,6 +746,7 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
|
|
|
746
746
|
else
|
|
747
747
|
# If size estimation is disabled we simply append the event and handle splitting later on (if needed)
|
|
748
748
|
append_event = true
|
|
749
|
+
add_bytes = 0
|
|
749
750
|
end
|
|
750
751
|
|
|
751
752
|
# if we haven't consumed the current event already
|
|
@@ -758,9 +759,12 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
|
|
|
758
759
|
|
|
759
760
|
}
|
|
760
761
|
|
|
761
|
-
# create a final request with any left over events
|
|
762
|
-
|
|
763
|
-
|
|
762
|
+
# create a final request with any left over events (and make sure there is at least one event)
|
|
763
|
+
if scalyr_events.size >= 1
|
|
764
|
+
multi_event_request = self.create_multi_event_request(scalyr_events, l_events, current_threads, logs)
|
|
765
|
+
multi_event_request_array << multi_event_request
|
|
766
|
+
end
|
|
767
|
+
|
|
764
768
|
multi_event_request_array
|
|
765
769
|
end
|
|
766
770
|
|
data/lib/scalyr/constants.rb
CHANGED
|
@@ -1,2 +1,2 @@
|
|
|
1
1
|
# encoding: utf-8
|
|
2
|
-
PLUGIN_VERSION = "v0.1.
|
|
2
|
+
PLUGIN_VERSION = "v0.1.24.beta"
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Gem::Specification.new do |s|
|
|
2
2
|
s.name = 'logstash-output-scalyr'
|
|
3
|
-
s.version = '0.1.
|
|
3
|
+
s.version = '0.1.24.beta'
|
|
4
4
|
s.licenses = ['Apache-2.0']
|
|
5
5
|
s.summary = "Scalyr output plugin for Logstash"
|
|
6
6
|
s.description = "Sends log data collected by Logstash to Scalyr (https://www.scalyr.com)"
|
|
@@ -392,6 +392,124 @@ describe LogStash::Outputs::Scalyr do
|
|
|
392
392
|
end
|
|
393
393
|
end
|
|
394
394
|
|
|
395
|
+
context "split large batches into multiple scalyr requests" do
|
|
396
|
+
it "estimate_each_event_size is true explicit (default) batch split into 3 scalyr requests" do
|
|
397
|
+
config = {
|
|
398
|
+
'api_write_token' => '1234',
|
|
399
|
+
'flatten_tags' => true,
|
|
400
|
+
'flat_tag_value' => 'true',
|
|
401
|
+
'flat_tag_prefix' => 'tag_prefix_',
|
|
402
|
+
'flatten_nested_values' => true, # this converts into string 'true'
|
|
403
|
+
'max_request_buffer' => 10,
|
|
404
|
+
'estimate_each_event_size' => true
|
|
405
|
+
}
|
|
406
|
+
plugin = LogStash::Outputs::Scalyr.new(config)
|
|
407
|
+
|
|
408
|
+
allow(plugin).to receive(:send_status).and_return(nil)
|
|
409
|
+
plugin.register
|
|
410
|
+
result = plugin.build_multi_event_request_array(sample_events)
|
|
411
|
+
expect(result.size).to eq(3)
|
|
412
|
+
|
|
413
|
+
body = JSON.parse(result[0][:body])
|
|
414
|
+
expect(body['events'].size).to eq(1)
|
|
415
|
+
|
|
416
|
+
body = JSON.parse(result[1][:body])
|
|
417
|
+
expect(body['events'].size).to eq(1)
|
|
418
|
+
|
|
419
|
+
body = JSON.parse(result[2][:body])
|
|
420
|
+
expect(body['events'].size).to eq(1)
|
|
421
|
+
expect(body['events'][0]['attrs']).to eq({
|
|
422
|
+
"nested_a" => 1,
|
|
423
|
+
"nested_b_0" => 3,
|
|
424
|
+
"nested_b_1" => 4,
|
|
425
|
+
"nested_b_2" => 5,
|
|
426
|
+
'seq' => 3,
|
|
427
|
+
'source_file' => 'my file 3',
|
|
428
|
+
'source_host' => 'my host 3',
|
|
429
|
+
'serverHost' => 'Logstash',
|
|
430
|
+
"tag_prefix_t1" => "true",
|
|
431
|
+
"tag_prefix_t2" => "true",
|
|
432
|
+
"tag_prefix_t3" => "true",
|
|
433
|
+
"parser" => "logstashParser",
|
|
434
|
+
})
|
|
435
|
+
end
|
|
436
|
+
|
|
437
|
+
it "estimate_each_event_size is true implicit (default) batch split into 3 scalyr requests" do
|
|
438
|
+
config = {
|
|
439
|
+
'api_write_token' => '1234',
|
|
440
|
+
'flatten_tags' => true,
|
|
441
|
+
'flat_tag_value' => 'true',
|
|
442
|
+
'flat_tag_prefix' => 'tag_prefix_',
|
|
443
|
+
'flatten_nested_values' => true, # this converts into string 'true'
|
|
444
|
+
'max_request_buffer' => 10,
|
|
445
|
+
}
|
|
446
|
+
plugin = LogStash::Outputs::Scalyr.new(config)
|
|
447
|
+
|
|
448
|
+
allow(plugin).to receive(:send_status).and_return(nil)
|
|
449
|
+
plugin.register
|
|
450
|
+
result = plugin.build_multi_event_request_array(sample_events)
|
|
451
|
+
expect(result.size).to eq(3)
|
|
452
|
+
|
|
453
|
+
body = JSON.parse(result[0][:body])
|
|
454
|
+
expect(body['events'].size).to eq(1)
|
|
455
|
+
|
|
456
|
+
body = JSON.parse(result[1][:body])
|
|
457
|
+
expect(body['events'].size).to eq(1)
|
|
458
|
+
|
|
459
|
+
body = JSON.parse(result[2][:body])
|
|
460
|
+
expect(body['events'].size).to eq(1)
|
|
461
|
+
expect(body['events'][0]['attrs']).to eq({
|
|
462
|
+
"nested_a" => 1,
|
|
463
|
+
"nested_b_0" => 3,
|
|
464
|
+
"nested_b_1" => 4,
|
|
465
|
+
"nested_b_2" => 5,
|
|
466
|
+
'seq' => 3,
|
|
467
|
+
'source_file' => 'my file 3',
|
|
468
|
+
'source_host' => 'my host 3',
|
|
469
|
+
'serverHost' => 'Logstash',
|
|
470
|
+
"tag_prefix_t1" => "true",
|
|
471
|
+
"tag_prefix_t2" => "true",
|
|
472
|
+
"tag_prefix_t3" => "true",
|
|
473
|
+
"parser" => "logstashParser",
|
|
474
|
+
})
|
|
475
|
+
end
|
|
476
|
+
|
|
477
|
+
it "estimate_each_event_size is false batch not split into multiple scalyr requests" do
|
|
478
|
+
config = {
|
|
479
|
+
'api_write_token' => '1234',
|
|
480
|
+
'flatten_tags' => true,
|
|
481
|
+
'flat_tag_value' => 'true',
|
|
482
|
+
'flat_tag_prefix' => 'tag_prefix_',
|
|
483
|
+
'flatten_nested_values' => true, # this converts into string 'true'
|
|
484
|
+
'max_request_buffer' => 10,
|
|
485
|
+
'estimate_each_event_size' => false
|
|
486
|
+
}
|
|
487
|
+
plugin = LogStash::Outputs::Scalyr.new(config)
|
|
488
|
+
|
|
489
|
+
allow(plugin).to receive(:send_status).and_return(nil)
|
|
490
|
+
plugin.register
|
|
491
|
+
result = plugin.build_multi_event_request_array(sample_events)
|
|
492
|
+
expect(result.size).to eq(1)
|
|
493
|
+
|
|
494
|
+
body = JSON.parse(result[0][:body])
|
|
495
|
+
expect(body['events'].size).to eq(3)
|
|
496
|
+
expect(body['events'][2]['attrs']).to eq({
|
|
497
|
+
"nested_a" => 1,
|
|
498
|
+
"nested_b_0" => 3,
|
|
499
|
+
"nested_b_1" => 4,
|
|
500
|
+
"nested_b_2" => 5,
|
|
501
|
+
'seq' => 3,
|
|
502
|
+
'source_file' => 'my file 3',
|
|
503
|
+
'source_host' => 'my host 3',
|
|
504
|
+
'serverHost' => 'Logstash',
|
|
505
|
+
"tag_prefix_t1" => "true",
|
|
506
|
+
"tag_prefix_t2" => "true",
|
|
507
|
+
"tag_prefix_t3" => "true",
|
|
508
|
+
"parser" => "logstashParser",
|
|
509
|
+
})
|
|
510
|
+
end
|
|
511
|
+
end
|
|
512
|
+
|
|
395
513
|
context "when not configured to flatten values and tags" do
|
|
396
514
|
config = {
|
|
397
515
|
'api_write_token' => '1234',
|