logstash-output-scalyr 0.2.7.beta → 0.2.8.beta

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: '098686f8c748dbe1d02cec87e92aa49df5db05049ff0d87ccd6fbb6271ccd797'
4
- data.tar.gz: '0073297594f936703190252584559ae60cd81a7b3807f1b2685b6966fece5610'
3
+ metadata.gz: da91d80d418081307dc1a6b756f17024712c0eab9a6d527a140288976a58b76f
4
+ data.tar.gz: 2ac792b42939edb371614c73c4dbb50fcc835f199d03685ff619d9d355c6465a
5
5
  SHA512:
6
- metadata.gz: 507ddc7c825ae4612907c4ad451597bf516732b389b5a628598f8eec12e8cdc1f7c4b5b71533803665d1fd18b3fce43a0aa3989729192eea26b2948273f9d267
7
- data.tar.gz: 310a452e0df0b74081a061030322b3f2a013bad233ba99b424fcea428af336f86370cb76bda62c06c5edcbee81731e5a498e137daf4f7d42a843ef988a8664df
6
+ metadata.gz: 0a87ca7115a2eb3f03282046510bd51869e575eb29b8bec6ee273cbc9f68fea0167890835ffdda8bd0d571ac4299aff689642542d8dd5d88eaa2a89fe760a3f0
7
+ data.tar.gz: 7457fff5003d10b7288a9a958f3ec3f4a490c19da51a3ac3ade0402a37f74fafffc2501185857b4b3a7263753cb6af3276471724870efa0dca9e0773671b2f3f
data/CHANGELOG.md CHANGED
@@ -1,5 +1,21 @@
1
1
  # Beta
2
2
 
3
+ ## 0.2.8.beta
4
+
5
+ * Update ``.gemspec`` gem metadata to not include ``spec/`` directory with the tests and tests
6
+ fixtures with the actual production gem file.
7
+
8
+ * Do not retry requests that will never be accepted by the server.
9
+ Specifically, any request that returns HTTP Status code 413 is too large, and
10
+ will never be accepted. Instead of simply retrying for 10 minutes before
11
+ sending the request to the DLQ, skip the retries go directly to sending the
12
+ request to the DLQ.
13
+
14
+ To be notified when an event fails to be ingested for whatever reason, create
15
+ an alert using the query: ``parser='logstash_plugin_metrics'
16
+ failed_events_processed > 0``. Instructions on how to create an alert can be
17
+ found in our docs here: https://scalyr.com/help/alerts
18
+
3
19
  ## 0.2.7.beta
4
20
 
5
21
  * SSL cert validation code has been simplified. Now ``ssl_ca_bundle_path`` config option
data/README.md CHANGED
@@ -505,10 +505,9 @@ To deploy the current code on your machine run these commands:
505
505
 
506
506
  ```
507
507
  rm -rf vendor/
508
- bundle check --path vendor/bundle || bundle install --deployment
508
+ bundle install
509
509
  curl -u RUBY_USER:RUBY_PASSWORD https://rubygems.org/api/v1/api_key.yaml > ~/.gem/credentials
510
510
  chmod 0600 ~/.gem/credentials
511
- bundle exec rake vendor
512
511
  bundle exec rspec
513
512
  bundle exec rake publish_gem
514
513
  ```
@@ -518,10 +517,9 @@ Or as an alternative if ``rake publish_gem`` task doesn't appear to work for wha
518
517
 
519
518
  ```
520
519
  rm -rf vendor/
521
- bundle check --path vendor/bundle || bundle install --deployment
520
+ bundle install
522
521
  curl -u RUBY_USER:RUBY_PASSWORD https://rubygems.org/api/v1/api_key.yaml > ~/.gem/credentials
523
522
  chmod 0600 ~/.gem/credentials
524
- bundle exec rake vendor
525
523
  bundle exec rspec
526
524
  rvm use jruby
527
525
  bundle exec gem build logstash-output-scalyr.gemspec
@@ -500,6 +500,24 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
500
500
  result.push(multi_event_request)
501
501
  end
502
502
 
503
+ rescue Scalyr::Common::Client::PayloadTooLargeError => e
504
+ # if the payload is too large, we do not retry. we send to DLQ or drop it.
505
+ exc_data = {
506
+ :error_class => e.e_class,
507
+ :url => e.url.to_s,
508
+ :message => e.message,
509
+ :batch_num => batch_num,
510
+ :total_batches => total_batches,
511
+ :record_count => multi_event_request[:record_count],
512
+ :payload_size => multi_event_request[:body].bytesize,
513
+ }
514
+ exc_data[:code] = e.code if e.code
515
+ if defined?(e.body) and e.body
516
+ exc_data[:body] = Scalyr::Common::Util.truncate(e.body, 512)
517
+ end
518
+ exc_data[:payload] = "\tSample payload: #{multi_event_request[:body][0,1024]}..."
519
+ log_retry_failure(multi_event_request, exc_data, 0, 0)
520
+ next
503
521
  rescue Scalyr::Common::Client::ServerError, Scalyr::Common::Client::ClientError => e
504
522
  sleep_interval = sleep_for(sleep_interval)
505
523
  exc_sleep += sleep_interval
@@ -634,18 +652,25 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
634
652
  @multi_receive_statistics[:total_events_processed] += multi_event_request[:logstash_events].length
635
653
  @multi_receive_statistics[:failed_events_processed] += multi_event_request[:logstash_events].length
636
654
  end
637
- message = "Failed to send #{multi_event_request[:logstash_events].length} events after #{exc_retries} tries."
638
655
  sample_events = Array.new
639
656
  multi_event_request[:logstash_events][0,5].each {|l_event|
640
657
  sample_events << Scalyr::Common::Util.truncate(l_event.to_hash.to_json, 256)
641
658
  }
642
- @logger.error(message, :error_data => exc_data, :sample_events => sample_events, :retries => exc_retries, :sleep_time => exc_sleep)
659
+ if exc_data[:code] == 413
660
+ message = "Failed to send #{multi_event_request[:logstash_events].length} events due to exceeding maximum request size. Not retrying non-retriable request."
661
+ # For PayloadTooLargeError we already include sample Scalyr payload in exc_data so there is no need
662
+ # to include redundant sample Logstash event objects
663
+ @logger.error(message, :error_data => exc_data)
664
+ else
665
+ message = "Failed to send #{multi_event_request[:logstash_events].length} events after #{exc_retries} tries."
666
+ @logger.error(message, :error_data => exc_data, :sample_events => sample_events, :retries => exc_retries, :sleep_time => exc_sleep)
667
+ end
643
668
  if @dlq_writer
644
669
  multi_event_request[:logstash_events].each {|l_event|
645
670
  @dlq_writer.write(l_event, "#{exc_data[:message]}")
646
671
  }
647
672
  else
648
- @logger.warn("Dead letter queue not configured, dropping #{multi_event_request[:logstash_events].length} events after #{exc_retries} tries.", :sample_events => sample_events)
673
+ @logger.warn("Dead letter queue not configured, dropping #{multi_event_request[:logstash_events].length} events.", :sample_events => sample_events)
649
674
  end
650
675
  end
651
676
 
@@ -1050,7 +1075,7 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
1050
1075
  serialized_request_size = serialized_body.bytesize
1051
1076
 
1052
1077
  # We give it "buffer" since the splitting code allows for some slack and doesn't take into account top-level non-event attributes
1053
- if not @estimate_each_event_size and serialized_request_size >= @max_request_buffer + 5000
1078
+ if serialized_request_size >= @max_request_buffer + 5000
1054
1079
  # TODO: If we end up here is estimate config opsion is false, split the request here into multiple ones
1055
1080
  @logger.warn("Serialized request size (#{serialized_request_size}) is larger than max_request_buffer (#{max_request_buffer})!")
1056
1081
  end
@@ -27,6 +27,18 @@ end
27
27
  # An exception that signifies the Scalyr server received the upload request but dropped it
28
28
  #---------------------------------------------------------------------------------------------------------------------
29
29
  class RequestDroppedError < ServerError;
30
+ def initialize(msg=nil, code=nil, url=nil, body=nil, e_class="Scalyr::Common::Client::RequestDroppedError")
31
+ super(msg, code, url, body, e_class)
32
+ end
33
+ end
34
+
35
+ #---------------------------------------------------------------------------------------------------------------------
36
+ # An exception that signifies the Scalyr server received the upload request but dropped it due to it being too large.
37
+ #---------------------------------------------------------------------------------------------------------------------
38
+ class PayloadTooLargeError < ServerError;
39
+ def initialize(msg=nil, code=nil, url=nil, body=nil, e_class="Scalyr::Common::Client::PayloadTooLargeError")
40
+ super(msg, code, url, body, e_class)
41
+ end
30
42
  end
31
43
 
32
44
  #---------------------------------------------------------------------------------------------------------------------
@@ -308,15 +320,17 @@ class ClientSession
308
320
  end
309
321
 
310
322
  status = response_hash["status"]
323
+ code = response.code.to_s.strip.to_i
311
324
 
312
325
  if status != "success"
313
- if status =~ /discardBuffer/
326
+ if code == 413
327
+ raise PayloadTooLargeError.new(status, response.code, @add_events_uri, response.body)
328
+ elsif status =~ /discardBuffer/
314
329
  raise RequestDroppedError.new(status, response.code, @add_events_uri, response.body)
315
330
  else
316
331
  raise ServerError.new(status, response.code, @add_events_uri, response.body)
317
332
  end
318
333
  else
319
- code = response.code.to_s.strip.to_i
320
334
  if code < 200 or code > 299
321
335
  raise ServerError.new(status, response.code, @add_events_uri, response.body)
322
336
  end
@@ -1,6 +1,6 @@
1
1
  # encoding: utf-8
2
2
 
3
- PLUGIN_VERSION = "v0.2.7.beta"
3
+ PLUGIN_VERSION = "v0.2.8.beta"
4
4
 
5
5
  # Special event level attribute name which can be used for setting event level serverHost attribute
6
6
  EVENT_LEVEL_SERVER_HOST_ATTRIBUTE_NAME = '__origServerHost'
@@ -1,6 +1,6 @@
1
1
  Gem::Specification.new do |s|
2
2
  s.name = 'logstash-output-scalyr'
3
- s.version = '0.2.7.beta'
3
+ s.version = '0.2.8.beta'
4
4
  s.licenses = ['Apache-2.0']
5
5
  s.summary = "Scalyr output plugin for Logstash"
6
6
  s.description = "Sends log data collected by Logstash to Scalyr (https://www.scalyr.com)"
@@ -10,7 +10,7 @@ Gem::Specification.new do |s|
10
10
  s.require_paths = ["lib"]
11
11
 
12
12
  # Files
13
- s.files = Dir['lib/**/*','spec/**/*','*.gemspec','*.md','CONTRIBUTORS','Gemfile','LICENSE','NOTICE.TXT']
13
+ s.files = Dir['lib/**/*','*.gemspec','*.md','CONTRIBUTORS','Gemfile','LICENSE','NOTICE.TXT']
14
14
  # Tests
15
15
  s.test_files = s.files.grep(%r{^(test|spec|features)/})
16
16
 
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: logstash-output-scalyr
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.2.7.beta
4
+ version: 0.2.8.beta
5
5
  platform: ruby
6
6
  authors:
7
7
  - Edward Chee
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2022-08-04 00:00:00.000000000 Z
11
+ date: 2022-10-20 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  requirement: !ruby/object:Gem::Requirement
@@ -165,16 +165,6 @@ files:
165
165
  - lib/scalyr/common/util.rb
166
166
  - lib/scalyr/constants.rb
167
167
  - logstash-output-scalyr.gemspec
168
- - spec/benchmarks/bignum_fixing.rb
169
- - spec/benchmarks/flattening_and_serialization.rb
170
- - spec/benchmarks/json_serialization.rb
171
- - spec/benchmarks/metrics_overhead.rb
172
- - spec/benchmarks/set_session_level_serverhost_on_events.rb
173
- - spec/benchmarks/util.rb
174
- - spec/logstash/outputs/fixtures/example_com.pem
175
- - spec/logstash/outputs/scalyr_integration_spec.rb
176
- - spec/logstash/outputs/scalyr_spec.rb
177
- - spec/scalyr/common/util_spec.rb
178
168
  homepage: https://www.scalyr.com/help/data-sources#logstash
179
169
  licenses:
180
170
  - Apache-2.0
@@ -201,14 +191,4 @@ rubygems_version: 2.7.10
201
191
  signing_key:
202
192
  specification_version: 4
203
193
  summary: Scalyr output plugin for Logstash
204
- test_files:
205
- - spec/benchmarks/bignum_fixing.rb
206
- - spec/benchmarks/flattening_and_serialization.rb
207
- - spec/benchmarks/json_serialization.rb
208
- - spec/benchmarks/metrics_overhead.rb
209
- - spec/benchmarks/set_session_level_serverhost_on_events.rb
210
- - spec/benchmarks/util.rb
211
- - spec/logstash/outputs/fixtures/example_com.pem
212
- - spec/logstash/outputs/scalyr_integration_spec.rb
213
- - spec/logstash/outputs/scalyr_spec.rb
214
- - spec/scalyr/common/util_spec.rb
194
+ test_files: []
@@ -1,87 +0,0 @@
1
- require 'benchmark'
2
- require 'quantile'
3
-
4
- require_relative '../../lib/scalyr/common/util'
5
- require_relative './util'
6
-
7
- # Micro benchmark which measures how long it takes to find all the Bignums in a record and convert them to strings
8
-
9
- ITERATIONS = 500
10
-
11
- def rand_bignum()
12
- return 200004000020304050300 + rand(999999)
13
- end
14
-
15
- def generate_hash(widths)
16
- result = {}
17
- if widths.empty?
18
- return rand_bignum()
19
- else
20
- widths[0].times do
21
- result[rand_str(9)] = generate_hash(widths[1..widths.length])
22
- end
23
- return result
24
- end
25
- end
26
-
27
- def generate_data_array_for_spec(spec)
28
- data = []
29
- ITERATIONS.times do
30
- data << generate_hash(spec)
31
- end
32
-
33
- data
34
- end
35
-
36
- def run_benchmark_and_print_results(data, run_benchmark_func)
37
- puts ""
38
- puts "Using %s total keys in a hash" % [Scalyr::Common::Util.flatten(data[0]).count]
39
- puts ""
40
-
41
- result = []
42
- ITERATIONS.times do |i|
43
- result << Benchmark.measure { run_benchmark_func.(data[i]) }
44
- end
45
-
46
- sum = result.inject(nil) { |sum, t| sum.nil? ? sum = t : sum += t }
47
- avg = sum / result.size
48
-
49
- Benchmark.bm(7, "sum:", "avg:") do |b|
50
- [sum, avg]
51
- end
52
- puts ""
53
- end
54
-
55
-
56
- puts "Using %s iterations" % [ITERATIONS]
57
- puts ""
58
-
59
- @value = Quantile::Estimator.new
60
- @prng = Random.new
61
-
62
- def convert_bignums(record)
63
- Scalyr::Common::Util.convert_bignums(record)
64
- end
65
-
66
- puts "Util.convert_bignums()"
67
- puts "==============================="
68
-
69
- # Around ~200 keys in a hash
70
- data = generate_data_array_for_spec([4, 4, 3, 4])
71
- run_benchmark_and_print_results(data, method(:convert_bignums))
72
-
73
- # Around ~200 keys in a hash (single level)
74
- data = generate_data_array_for_spec([200])
75
- run_benchmark_and_print_results(data, method(:convert_bignums))
76
-
77
- # Around ~512 keys in a hash
78
- data = generate_data_array_for_spec([8, 4, 4, 4])
79
- run_benchmark_and_print_results(data, method(:convert_bignums))
80
-
81
- # Around ~960 keys in a hash
82
- data = generate_data_array_for_spec([12, 5, 4, 4])
83
- run_benchmark_and_print_results(data, method(:convert_bignums))
84
-
85
- # Around ~2700 keys in a hash
86
- data = generate_data_array_for_spec([14, 8, 6, 4])
87
- run_benchmark_and_print_results(data, method(:convert_bignums))
@@ -1,100 +0,0 @@
1
- require 'benchmark'
2
- require 'json'
3
-
4
- require_relative '../../lib/scalyr/common/util'
5
- require_relative './util'
6
-
7
- # NOTE: When using jRuby using multiple iterations with the same dataset doesn't make
8
- # sense since it will just use JITed version of the code which will be very fast. If we
9
- # wanted to accurately measure using multiple iterations we would need te different
10
- # input data for each iteration.
11
- ITERATIONS = 500
12
-
13
- def run_benchmark_and_print_results(data, run_benchmark_func)
14
- puts ""
15
- puts "Using %s total keys in a hash" % [Scalyr::Common::Util.flatten(data[0]).count]
16
- puts ""
17
-
18
- result = []
19
- ITERATIONS.times do |i|
20
- result << Benchmark.measure { run_benchmark_func.(data[i]) }
21
- end
22
-
23
- sum = result.inject(nil) { |sum, t| sum.nil? ? sum = t : sum += t }
24
- avg = sum / result.size
25
-
26
- Benchmark.bm(7, "sum:", "avg:") do |b|
27
- [sum, avg]
28
- end
29
- puts ""
30
- end
31
-
32
- def flatten_data_func(data)
33
- Scalyr::Common::Util.flatten(data)
34
- end
35
-
36
- def json_serialize_data(data)
37
- data.to_json
38
- end
39
-
40
- DATASETS = {
41
- :keys_50 => generate_data_array_for_spec([3, 3, 3, 2]),
42
- :keys_200 => generate_data_array_for_spec([4, 4, 3, 4]),
43
- :keys_200_flat => generate_data_array_for_spec([200]),
44
- :keys_512 => generate_data_array_for_spec([8, 4, 4, 4]),
45
- :keys_960 => generate_data_array_for_spec([12, 5, 4, 4]),
46
- :keys_2700 => generate_data_array_for_spec([14, 8, 6, 4])
47
- }
48
-
49
- puts "Using %s iterations" % [ITERATIONS]
50
- puts ""
51
-
52
- puts "Scalyr::Common::Util.flatten()"
53
- puts "==============================="
54
-
55
- # Around ~50 keys in a hash
56
- data = DATASETS[:keys_50]
57
- run_benchmark_and_print_results(data, method(:flatten_data_func))
58
-
59
- # Around ~200 keys in a hash
60
- data = DATASETS[:keys_200]
61
- run_benchmark_and_print_results(data, method(:flatten_data_func))
62
-
63
- # Around ~200 keys in a hash (single level)
64
- data = DATASETS[:keys_200_flat]
65
- run_benchmark_and_print_results(data, method(:flatten_data_func))
66
-
67
- # Around ~512 keys in a hash
68
- data = DATASETS[:keys_512]
69
- run_benchmark_and_print_results(data, method(:flatten_data_func))
70
-
71
- # Around ~960 keys in a hash
72
- data = DATASETS[:keys_960]
73
- run_benchmark_and_print_results(data, method(:flatten_data_func))
74
-
75
- # Around ~2700 keys in a hash
76
- data = DATASETS[:keys_2700]
77
- run_benchmark_and_print_results(data, method(:flatten_data_func))
78
-
79
- puts "JSON.dumps (hash.to_dict)"
80
- puts "==============================="
81
-
82
- # Around ~200 keys in a hash
83
- data = generate_data_array_for_spec([4, 4, 3, 4])
84
- run_benchmark_and_print_results(data, method(:json_serialize_data))
85
-
86
- # Around ~200 keys in a hash (single level)
87
- data = DATASETS[:keys_200_flat]
88
- run_benchmark_and_print_results(data, method(:json_serialize_data))
89
-
90
- # Around ~512 keys in a hash
91
- data = generate_data_array_for_spec([8, 4, 4, 4])
92
- run_benchmark_and_print_results(data, method(:json_serialize_data))
93
-
94
- # Around ~960 keys in a hash
95
- data = generate_data_array_for_spec([12, 5, 4, 4])
96
- run_benchmark_and_print_results(data, method(:json_serialize_data))
97
-
98
- # Around ~2700 keys in a hash
99
- data = generate_data_array_for_spec([14, 8, 6, 4])
100
- run_benchmark_and_print_results(data, method(:json_serialize_data))
@@ -1,85 +0,0 @@
1
- require 'benchmark'
2
- require 'json'
3
- require 'jrjackson'
4
-
5
- require_relative '../../lib/scalyr/common/util'
6
- require_relative './util'
7
-
8
- ITERATIONS = 500
9
-
10
- def json_serialize_data_native(data)
11
- data.to_json
12
- end
13
-
14
- def json_serialize_data_jrjackson(data)
15
- JrJackson::Json.dump(data)
16
- end
17
-
18
- DATASETS = {
19
- :keys_50 => generate_data_array_for_spec([3, 3, 3, 2]),
20
- :keys_200 => generate_data_array_for_spec([4, 4, 3, 4]),
21
- :keys_200_flat => generate_data_array_for_spec([200]),
22
- :keys_512 => generate_data_array_for_spec([8, 4, 4, 4]),
23
- :keys_960 => generate_data_array_for_spec([12, 5, 4, 4]),
24
- :keys_2700 => generate_data_array_for_spec([14, 8, 6, 4])
25
- }
26
-
27
- def run_benchmark_and_print_results(data, run_benchmark_func)
28
- puts ""
29
- puts "Using %s total keys in a hash" % [Scalyr::Common::Util.flatten(data[0]).count]
30
- puts ""
31
-
32
- result = []
33
- ITERATIONS.times do |i|
34
- result << Benchmark.measure { run_benchmark_func.(data[i]) }
35
- end
36
-
37
- sum = result.inject(nil) { |sum, t| sum.nil? ? sum = t : sum += t }
38
- avg = sum / result.size
39
-
40
- Benchmark.bm(7, "sum:", "avg:") do |b|
41
- [sum, avg]
42
- end
43
- puts ""
44
- end
45
-
46
- puts "Using %s iterations" % [ITERATIONS]
47
- puts ""
48
-
49
- puts "native"
50
- puts "==============================="
51
-
52
- # Around ~50 keys in a hash
53
- data = DATASETS[:keys_50]
54
- run_benchmark_and_print_results(data, method(:json_serialize_data_native))
55
-
56
- # Around ~200 keys in a hash
57
- data = DATASETS[:keys_200]
58
- run_benchmark_and_print_results(data, method(:json_serialize_data_native))
59
-
60
- # Around ~200 keys in a hash (single level)
61
- data = DATASETS[:keys_200_flat]
62
- run_benchmark_and_print_results(data, method(:json_serialize_data_native))
63
-
64
- # Around ~2700 keys in a hash
65
- data = DATASETS[:keys_2700]
66
- run_benchmark_and_print_results(data, method(:json_serialize_data_native))
67
-
68
- puts "jrjackson"
69
- puts "==============================="
70
-
71
- # Around ~50 keys in a hash
72
- data = DATASETS[:keys_50]
73
- run_benchmark_and_print_results(data, method(:json_serialize_data_jrjackson))
74
-
75
- # Around ~200 keys in a hash
76
- data = DATASETS[:keys_200]
77
- run_benchmark_and_print_results(data, method(:json_serialize_data_jrjackson))
78
-
79
- # Around ~200 keys in a hash (single level)
80
- data = DATASETS[:keys_200_flat]
81
- run_benchmark_and_print_results(data, method(:json_serialize_data_jrjackson))
82
-
83
- # Around ~2700 keys in a hash
84
- data = DATASETS[:keys_2700]
85
- run_benchmark_and_print_results(data, method(:json_serialize_data_jrjackson))
@@ -1,48 +0,0 @@
1
- require 'benchmark'
2
- require 'quantile'
3
-
4
- require_relative '../../lib/scalyr/common/util'
5
-
6
- # Micro benchmark which measures how much overhead Quantile.observe adds vs random sampling to see
7
- # where making sampling (e.g. on event level metrics) is desired
8
-
9
- ITERATIONS = 10000
10
-
11
- def run_benchmark_and_print_results(run_benchmark_func)
12
- result = []
13
- ITERATIONS.times do |i|
14
- result << Benchmark.measure { run_benchmark_func.() }
15
- end
16
-
17
- sum = result.inject(nil) { |sum, t| sum.nil? ? sum = t : sum += t }
18
- avg = sum / result.size
19
-
20
- Benchmark.bm(7, "sum:", "avg:") do |b|
21
- [sum, avg]
22
- end
23
- puts ""
24
- end
25
-
26
-
27
- puts "Using %s iterations" % [ITERATIONS]
28
- puts ""
29
-
30
- @value = Quantile::Estimator.new
31
- @prng = Random.new
32
-
33
- def quantile_observe()
34
- @value.observe(5)
35
- end
36
-
37
- def random_sample()
38
- return @prng.rand(0.0..1.0) < 0.5
39
- end
40
-
41
- puts "Quartile.observe()"
42
- puts "==============================="
43
-
44
- run_benchmark_and_print_results(method(:quantile_observe))
45
-
46
- puts "random sample"
47
- puts "==============================="
48
- run_benchmark_and_print_results(method(:random_sample))
@@ -1,107 +0,0 @@
1
- require 'benchmark'
2
- require 'quantile'
3
-
4
- require_relative '../../lib/scalyr/constants'
5
- require_relative '../../lib/scalyr/common/util'
6
- require_relative './util'
7
-
8
- # Micro benchmark which measures how long "set_session_level_serverhost_on_events" takes
9
-
10
- ITERATIONS = 100
11
-
12
- def run_benchmark_and_print_results(data, run_benchmark_func)
13
- puts ""
14
- puts "Using %s total events in a batch" % [data[0].size]
15
- puts ""
16
-
17
- result = []
18
- ITERATIONS.times do |i|
19
- result << Benchmark.measure { run_benchmark_func.(data[i]) }
20
- end
21
-
22
- sum = result.inject(nil) { |sum, t| sum.nil? ? sum = t : sum += t }
23
- avg = sum / result.size
24
-
25
- Benchmark.bm(7, "sum:", "avg:") do |b|
26
- [sum, avg]
27
- end
28
- puts ""
29
- end
30
-
31
- # Generate random event with only single event having special server host attribute set which
32
- # represents a worst case scenario since we need to backfill rest of the events.
33
- def generate_events(count)
34
- result = []
35
-
36
- ITERATIONS.times do |iteration|
37
- events = []
38
-
39
- count.times do |index|
40
- event = generate_hash([2])
41
- event[:attrs] = Hash.new
42
- event[:log] = 1
43
-
44
- if index == count - 1
45
- event[:attrs][EVENT_LEVEL_SERVER_HOST_ATTRIBUTE_NAME] = format("test-host-%s", index)
46
- end
47
-
48
- events << event
49
- end
50
-
51
- raise "Assertion failed" unless events.size == count
52
-
53
- result << events
54
- end
55
-
56
- raise "Assertion failed" unless result.size == ITERATIONS
57
- result
58
- end
59
-
60
- def run_func(events)
61
- # NOTE: This function manipulates events in place
62
- events.each_with_index do |event, index|
63
- if index < events.size - 1
64
- # Last event will have _origServerHost set, but others won't
65
- raise "Assertion failed" unless event[:attrs][EVENT_LEVEL_SERVER_HOST_ATTRIBUTE_NAME].nil?
66
- end
67
- end
68
-
69
- Scalyr::Common::Util.set_session_level_serverhost_on_events("session-server-host-dummy", events, {}, true)
70
-
71
- events.each do |event|
72
- raise "Assertion failed" unless event[:attrs][EVENT_LEVEL_SERVER_HOST_ATTRIBUTE_NAME].nil? == false
73
- end
74
- end
75
-
76
-
77
- puts "Using %s iterations" % [ITERATIONS]
78
- puts ""
79
-
80
- @value = Quantile::Estimator.new
81
-
82
- puts "Util.set_session_level_serverhost_on_events()"
83
- puts "==============================="
84
-
85
- # 100 events in a batch
86
- data = generate_events(100)
87
- run_benchmark_and_print_results(data, method(:run_func))
88
-
89
- # 500 events in a batch
90
- data = generate_events(500)
91
- run_benchmark_and_print_results(data, method(:run_func))
92
-
93
- # 1000 events in a batch
94
- data = generate_events(1000)
95
- run_benchmark_and_print_results(data, method(:run_func))
96
-
97
- # 2000 events in a batch
98
- data = generate_events(2000)
99
- run_benchmark_and_print_results(data, method(:run_func))
100
-
101
- # 3000 events in a batch
102
- data = generate_events(3000)
103
- run_benchmark_and_print_results(data, method(:run_func))
104
-
105
- # 5000 events in a batch
106
- data = generate_events(5000)
107
- run_benchmark_and_print_results(data, method(:run_func))