logstash-output-scalyr 0.2.7.beta → 0.2.8.beta

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: '098686f8c748dbe1d02cec87e92aa49df5db05049ff0d87ccd6fbb6271ccd797'
4
- data.tar.gz: '0073297594f936703190252584559ae60cd81a7b3807f1b2685b6966fece5610'
3
+ metadata.gz: da91d80d418081307dc1a6b756f17024712c0eab9a6d527a140288976a58b76f
4
+ data.tar.gz: 2ac792b42939edb371614c73c4dbb50fcc835f199d03685ff619d9d355c6465a
5
5
  SHA512:
6
- metadata.gz: 507ddc7c825ae4612907c4ad451597bf516732b389b5a628598f8eec12e8cdc1f7c4b5b71533803665d1fd18b3fce43a0aa3989729192eea26b2948273f9d267
7
- data.tar.gz: 310a452e0df0b74081a061030322b3f2a013bad233ba99b424fcea428af336f86370cb76bda62c06c5edcbee81731e5a498e137daf4f7d42a843ef988a8664df
6
+ metadata.gz: 0a87ca7115a2eb3f03282046510bd51869e575eb29b8bec6ee273cbc9f68fea0167890835ffdda8bd0d571ac4299aff689642542d8dd5d88eaa2a89fe760a3f0
7
+ data.tar.gz: 7457fff5003d10b7288a9a958f3ec3f4a490c19da51a3ac3ade0402a37f74fafffc2501185857b4b3a7263753cb6af3276471724870efa0dca9e0773671b2f3f
data/CHANGELOG.md CHANGED
@@ -1,5 +1,21 @@
1
1
  # Beta
2
2
 
3
+ ## 0.2.8.beta
4
+
5
+ * Update ``.gemspec`` gem metadata to not include ``spec/`` directory with the tests and tests
6
+ fixtures with the actual production gem file.
7
+
8
+ * Do not retry requests that will never be accepted by the server.
9
+ Specifically, any request that returns HTTP Status code 413 is too large, and
10
+ will never be accepted. Instead of simply retrying for 10 minutes before
11
+ sending the request to the DLQ, skip the retries go directly to sending the
12
+ request to the DLQ.
13
+
14
+ To be notified when an event fails to be ingested for whatever reason, create
15
+ an alert using the query: ``parser='logstash_plugin_metrics'
16
+ failed_events_processed > 0``. Instructions on how to create an alert can be
17
+ found in our docs here: https://scalyr.com/help/alerts
18
+
3
19
  ## 0.2.7.beta
4
20
 
5
21
  * SSL cert validation code has been simplified. Now ``ssl_ca_bundle_path`` config option
data/README.md CHANGED
@@ -505,10 +505,9 @@ To deploy the current code on your machine run these commands:
505
505
 
506
506
  ```
507
507
  rm -rf vendor/
508
- bundle check --path vendor/bundle || bundle install --deployment
508
+ bundle install
509
509
  curl -u RUBY_USER:RUBY_PASSWORD https://rubygems.org/api/v1/api_key.yaml > ~/.gem/credentials
510
510
  chmod 0600 ~/.gem/credentials
511
- bundle exec rake vendor
512
511
  bundle exec rspec
513
512
  bundle exec rake publish_gem
514
513
  ```
@@ -518,10 +517,9 @@ Or as an alternative if ``rake publish_gem`` task doesn't appear to work for wha
518
517
 
519
518
  ```
520
519
  rm -rf vendor/
521
- bundle check --path vendor/bundle || bundle install --deployment
520
+ bundle install
522
521
  curl -u RUBY_USER:RUBY_PASSWORD https://rubygems.org/api/v1/api_key.yaml > ~/.gem/credentials
523
522
  chmod 0600 ~/.gem/credentials
524
- bundle exec rake vendor
525
523
  bundle exec rspec
526
524
  rvm use jruby
527
525
  bundle exec gem build logstash-output-scalyr.gemspec
@@ -500,6 +500,24 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
500
500
  result.push(multi_event_request)
501
501
  end
502
502
 
503
+ rescue Scalyr::Common::Client::PayloadTooLargeError => e
504
+ # if the payload is too large, we do not retry. we send to DLQ or drop it.
505
+ exc_data = {
506
+ :error_class => e.e_class,
507
+ :url => e.url.to_s,
508
+ :message => e.message,
509
+ :batch_num => batch_num,
510
+ :total_batches => total_batches,
511
+ :record_count => multi_event_request[:record_count],
512
+ :payload_size => multi_event_request[:body].bytesize,
513
+ }
514
+ exc_data[:code] = e.code if e.code
515
+ if defined?(e.body) and e.body
516
+ exc_data[:body] = Scalyr::Common::Util.truncate(e.body, 512)
517
+ end
518
+ exc_data[:payload] = "\tSample payload: #{multi_event_request[:body][0,1024]}..."
519
+ log_retry_failure(multi_event_request, exc_data, 0, 0)
520
+ next
503
521
  rescue Scalyr::Common::Client::ServerError, Scalyr::Common::Client::ClientError => e
504
522
  sleep_interval = sleep_for(sleep_interval)
505
523
  exc_sleep += sleep_interval
@@ -634,18 +652,25 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
634
652
  @multi_receive_statistics[:total_events_processed] += multi_event_request[:logstash_events].length
635
653
  @multi_receive_statistics[:failed_events_processed] += multi_event_request[:logstash_events].length
636
654
  end
637
- message = "Failed to send #{multi_event_request[:logstash_events].length} events after #{exc_retries} tries."
638
655
  sample_events = Array.new
639
656
  multi_event_request[:logstash_events][0,5].each {|l_event|
640
657
  sample_events << Scalyr::Common::Util.truncate(l_event.to_hash.to_json, 256)
641
658
  }
642
- @logger.error(message, :error_data => exc_data, :sample_events => sample_events, :retries => exc_retries, :sleep_time => exc_sleep)
659
+ if exc_data[:code] == 413
660
+ message = "Failed to send #{multi_event_request[:logstash_events].length} events due to exceeding maximum request size. Not retrying non-retriable request."
661
+ # For PayloadTooLargeError we already include sample Scalyr payload in exc_data so there is no need
662
+ # to include redundant sample Logstash event objects
663
+ @logger.error(message, :error_data => exc_data)
664
+ else
665
+ message = "Failed to send #{multi_event_request[:logstash_events].length} events after #{exc_retries} tries."
666
+ @logger.error(message, :error_data => exc_data, :sample_events => sample_events, :retries => exc_retries, :sleep_time => exc_sleep)
667
+ end
643
668
  if @dlq_writer
644
669
  multi_event_request[:logstash_events].each {|l_event|
645
670
  @dlq_writer.write(l_event, "#{exc_data[:message]}")
646
671
  }
647
672
  else
648
- @logger.warn("Dead letter queue not configured, dropping #{multi_event_request[:logstash_events].length} events after #{exc_retries} tries.", :sample_events => sample_events)
673
+ @logger.warn("Dead letter queue not configured, dropping #{multi_event_request[:logstash_events].length} events.", :sample_events => sample_events)
649
674
  end
650
675
  end
651
676
 
@@ -1050,7 +1075,7 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
1050
1075
  serialized_request_size = serialized_body.bytesize
1051
1076
 
1052
1077
  # We give it "buffer" since the splitting code allows for some slack and doesn't take into account top-level non-event attributes
1053
- if not @estimate_each_event_size and serialized_request_size >= @max_request_buffer + 5000
1078
+ if serialized_request_size >= @max_request_buffer + 5000
1054
1079
  # TODO: If we end up here is estimate config opsion is false, split the request here into multiple ones
1055
1080
  @logger.warn("Serialized request size (#{serialized_request_size}) is larger than max_request_buffer (#{max_request_buffer})!")
1056
1081
  end
@@ -27,6 +27,18 @@ end
27
27
  # An exception that signifies the Scalyr server received the upload request but dropped it
28
28
  #---------------------------------------------------------------------------------------------------------------------
29
29
  class RequestDroppedError < ServerError;
30
+ def initialize(msg=nil, code=nil, url=nil, body=nil, e_class="Scalyr::Common::Client::RequestDroppedError")
31
+ super(msg, code, url, body, e_class)
32
+ end
33
+ end
34
+
35
+ #---------------------------------------------------------------------------------------------------------------------
36
+ # An exception that signifies the Scalyr server received the upload request but dropped it due to it being too large.
37
+ #---------------------------------------------------------------------------------------------------------------------
38
+ class PayloadTooLargeError < ServerError;
39
+ def initialize(msg=nil, code=nil, url=nil, body=nil, e_class="Scalyr::Common::Client::PayloadTooLargeError")
40
+ super(msg, code, url, body, e_class)
41
+ end
30
42
  end
31
43
 
32
44
  #---------------------------------------------------------------------------------------------------------------------
@@ -308,15 +320,17 @@ class ClientSession
308
320
  end
309
321
 
310
322
  status = response_hash["status"]
323
+ code = response.code.to_s.strip.to_i
311
324
 
312
325
  if status != "success"
313
- if status =~ /discardBuffer/
326
+ if code == 413
327
+ raise PayloadTooLargeError.new(status, response.code, @add_events_uri, response.body)
328
+ elsif status =~ /discardBuffer/
314
329
  raise RequestDroppedError.new(status, response.code, @add_events_uri, response.body)
315
330
  else
316
331
  raise ServerError.new(status, response.code, @add_events_uri, response.body)
317
332
  end
318
333
  else
319
- code = response.code.to_s.strip.to_i
320
334
  if code < 200 or code > 299
321
335
  raise ServerError.new(status, response.code, @add_events_uri, response.body)
322
336
  end
@@ -1,6 +1,6 @@
1
1
  # encoding: utf-8
2
2
 
3
- PLUGIN_VERSION = "v0.2.7.beta"
3
+ PLUGIN_VERSION = "v0.2.8.beta"
4
4
 
5
5
  # Special event level attribute name which can be used for setting event level serverHost attribute
6
6
  EVENT_LEVEL_SERVER_HOST_ATTRIBUTE_NAME = '__origServerHost'
@@ -1,6 +1,6 @@
1
1
  Gem::Specification.new do |s|
2
2
  s.name = 'logstash-output-scalyr'
3
- s.version = '0.2.7.beta'
3
+ s.version = '0.2.8.beta'
4
4
  s.licenses = ['Apache-2.0']
5
5
  s.summary = "Scalyr output plugin for Logstash"
6
6
  s.description = "Sends log data collected by Logstash to Scalyr (https://www.scalyr.com)"
@@ -10,7 +10,7 @@ Gem::Specification.new do |s|
10
10
  s.require_paths = ["lib"]
11
11
 
12
12
  # Files
13
- s.files = Dir['lib/**/*','spec/**/*','*.gemspec','*.md','CONTRIBUTORS','Gemfile','LICENSE','NOTICE.TXT']
13
+ s.files = Dir['lib/**/*','*.gemspec','*.md','CONTRIBUTORS','Gemfile','LICENSE','NOTICE.TXT']
14
14
  # Tests
15
15
  s.test_files = s.files.grep(%r{^(test|spec|features)/})
16
16
 
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: logstash-output-scalyr
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.2.7.beta
4
+ version: 0.2.8.beta
5
5
  platform: ruby
6
6
  authors:
7
7
  - Edward Chee
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2022-08-04 00:00:00.000000000 Z
11
+ date: 2022-10-20 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  requirement: !ruby/object:Gem::Requirement
@@ -165,16 +165,6 @@ files:
165
165
  - lib/scalyr/common/util.rb
166
166
  - lib/scalyr/constants.rb
167
167
  - logstash-output-scalyr.gemspec
168
- - spec/benchmarks/bignum_fixing.rb
169
- - spec/benchmarks/flattening_and_serialization.rb
170
- - spec/benchmarks/json_serialization.rb
171
- - spec/benchmarks/metrics_overhead.rb
172
- - spec/benchmarks/set_session_level_serverhost_on_events.rb
173
- - spec/benchmarks/util.rb
174
- - spec/logstash/outputs/fixtures/example_com.pem
175
- - spec/logstash/outputs/scalyr_integration_spec.rb
176
- - spec/logstash/outputs/scalyr_spec.rb
177
- - spec/scalyr/common/util_spec.rb
178
168
  homepage: https://www.scalyr.com/help/data-sources#logstash
179
169
  licenses:
180
170
  - Apache-2.0
@@ -201,14 +191,4 @@ rubygems_version: 2.7.10
201
191
  signing_key:
202
192
  specification_version: 4
203
193
  summary: Scalyr output plugin for Logstash
204
- test_files:
205
- - spec/benchmarks/bignum_fixing.rb
206
- - spec/benchmarks/flattening_and_serialization.rb
207
- - spec/benchmarks/json_serialization.rb
208
- - spec/benchmarks/metrics_overhead.rb
209
- - spec/benchmarks/set_session_level_serverhost_on_events.rb
210
- - spec/benchmarks/util.rb
211
- - spec/logstash/outputs/fixtures/example_com.pem
212
- - spec/logstash/outputs/scalyr_integration_spec.rb
213
- - spec/logstash/outputs/scalyr_spec.rb
214
- - spec/scalyr/common/util_spec.rb
194
+ test_files: []
@@ -1,87 +0,0 @@
1
- require 'benchmark'
2
- require 'quantile'
3
-
4
- require_relative '../../lib/scalyr/common/util'
5
- require_relative './util'
6
-
7
- # Micro benchmark which measures how long it takes to find all the Bignums in a record and convert them to strings
8
-
9
- ITERATIONS = 500
10
-
11
- def rand_bignum()
12
- return 200004000020304050300 + rand(999999)
13
- end
14
-
15
- def generate_hash(widths)
16
- result = {}
17
- if widths.empty?
18
- return rand_bignum()
19
- else
20
- widths[0].times do
21
- result[rand_str(9)] = generate_hash(widths[1..widths.length])
22
- end
23
- return result
24
- end
25
- end
26
-
27
- def generate_data_array_for_spec(spec)
28
- data = []
29
- ITERATIONS.times do
30
- data << generate_hash(spec)
31
- end
32
-
33
- data
34
- end
35
-
36
- def run_benchmark_and_print_results(data, run_benchmark_func)
37
- puts ""
38
- puts "Using %s total keys in a hash" % [Scalyr::Common::Util.flatten(data[0]).count]
39
- puts ""
40
-
41
- result = []
42
- ITERATIONS.times do |i|
43
- result << Benchmark.measure { run_benchmark_func.(data[i]) }
44
- end
45
-
46
- sum = result.inject(nil) { |sum, t| sum.nil? ? sum = t : sum += t }
47
- avg = sum / result.size
48
-
49
- Benchmark.bm(7, "sum:", "avg:") do |b|
50
- [sum, avg]
51
- end
52
- puts ""
53
- end
54
-
55
-
56
- puts "Using %s iterations" % [ITERATIONS]
57
- puts ""
58
-
59
- @value = Quantile::Estimator.new
60
- @prng = Random.new
61
-
62
- def convert_bignums(record)
63
- Scalyr::Common::Util.convert_bignums(record)
64
- end
65
-
66
- puts "Util.convert_bignums()"
67
- puts "==============================="
68
-
69
- # Around ~200 keys in a hash
70
- data = generate_data_array_for_spec([4, 4, 3, 4])
71
- run_benchmark_and_print_results(data, method(:convert_bignums))
72
-
73
- # Around ~200 keys in a hash (single level)
74
- data = generate_data_array_for_spec([200])
75
- run_benchmark_and_print_results(data, method(:convert_bignums))
76
-
77
- # Around ~512 keys in a hash
78
- data = generate_data_array_for_spec([8, 4, 4, 4])
79
- run_benchmark_and_print_results(data, method(:convert_bignums))
80
-
81
- # Around ~960 keys in a hash
82
- data = generate_data_array_for_spec([12, 5, 4, 4])
83
- run_benchmark_and_print_results(data, method(:convert_bignums))
84
-
85
- # Around ~2700 keys in a hash
86
- data = generate_data_array_for_spec([14, 8, 6, 4])
87
- run_benchmark_and_print_results(data, method(:convert_bignums))
@@ -1,100 +0,0 @@
1
- require 'benchmark'
2
- require 'json'
3
-
4
- require_relative '../../lib/scalyr/common/util'
5
- require_relative './util'
6
-
7
- # NOTE: When using jRuby using multiple iterations with the same dataset doesn't make
8
- # sense since it will just use JITed version of the code which will be very fast. If we
9
- # wanted to accurately measure using multiple iterations we would need te different
10
- # input data for each iteration.
11
- ITERATIONS = 500
12
-
13
- def run_benchmark_and_print_results(data, run_benchmark_func)
14
- puts ""
15
- puts "Using %s total keys in a hash" % [Scalyr::Common::Util.flatten(data[0]).count]
16
- puts ""
17
-
18
- result = []
19
- ITERATIONS.times do |i|
20
- result << Benchmark.measure { run_benchmark_func.(data[i]) }
21
- end
22
-
23
- sum = result.inject(nil) { |sum, t| sum.nil? ? sum = t : sum += t }
24
- avg = sum / result.size
25
-
26
- Benchmark.bm(7, "sum:", "avg:") do |b|
27
- [sum, avg]
28
- end
29
- puts ""
30
- end
31
-
32
- def flatten_data_func(data)
33
- Scalyr::Common::Util.flatten(data)
34
- end
35
-
36
- def json_serialize_data(data)
37
- data.to_json
38
- end
39
-
40
- DATASETS = {
41
- :keys_50 => generate_data_array_for_spec([3, 3, 3, 2]),
42
- :keys_200 => generate_data_array_for_spec([4, 4, 3, 4]),
43
- :keys_200_flat => generate_data_array_for_spec([200]),
44
- :keys_512 => generate_data_array_for_spec([8, 4, 4, 4]),
45
- :keys_960 => generate_data_array_for_spec([12, 5, 4, 4]),
46
- :keys_2700 => generate_data_array_for_spec([14, 8, 6, 4])
47
- }
48
-
49
- puts "Using %s iterations" % [ITERATIONS]
50
- puts ""
51
-
52
- puts "Scalyr::Common::Util.flatten()"
53
- puts "==============================="
54
-
55
- # Around ~50 keys in a hash
56
- data = DATASETS[:keys_50]
57
- run_benchmark_and_print_results(data, method(:flatten_data_func))
58
-
59
- # Around ~200 keys in a hash
60
- data = DATASETS[:keys_200]
61
- run_benchmark_and_print_results(data, method(:flatten_data_func))
62
-
63
- # Around ~200 keys in a hash (single level)
64
- data = DATASETS[:keys_200_flat]
65
- run_benchmark_and_print_results(data, method(:flatten_data_func))
66
-
67
- # Around ~512 keys in a hash
68
- data = DATASETS[:keys_512]
69
- run_benchmark_and_print_results(data, method(:flatten_data_func))
70
-
71
- # Around ~960 keys in a hash
72
- data = DATASETS[:keys_960]
73
- run_benchmark_and_print_results(data, method(:flatten_data_func))
74
-
75
- # Around ~2700 keys in a hash
76
- data = DATASETS[:keys_2700]
77
- run_benchmark_and_print_results(data, method(:flatten_data_func))
78
-
79
- puts "JSON.dumps (hash.to_dict)"
80
- puts "==============================="
81
-
82
- # Around ~200 keys in a hash
83
- data = generate_data_array_for_spec([4, 4, 3, 4])
84
- run_benchmark_and_print_results(data, method(:json_serialize_data))
85
-
86
- # Around ~200 keys in a hash (single level)
87
- data = DATASETS[:keys_200_flat]
88
- run_benchmark_and_print_results(data, method(:json_serialize_data))
89
-
90
- # Around ~512 keys in a hash
91
- data = generate_data_array_for_spec([8, 4, 4, 4])
92
- run_benchmark_and_print_results(data, method(:json_serialize_data))
93
-
94
- # Around ~960 keys in a hash
95
- data = generate_data_array_for_spec([12, 5, 4, 4])
96
- run_benchmark_and_print_results(data, method(:json_serialize_data))
97
-
98
- # Around ~2700 keys in a hash
99
- data = generate_data_array_for_spec([14, 8, 6, 4])
100
- run_benchmark_and_print_results(data, method(:json_serialize_data))
@@ -1,85 +0,0 @@
1
- require 'benchmark'
2
- require 'json'
3
- require 'jrjackson'
4
-
5
- require_relative '../../lib/scalyr/common/util'
6
- require_relative './util'
7
-
8
- ITERATIONS = 500
9
-
10
- def json_serialize_data_native(data)
11
- data.to_json
12
- end
13
-
14
- def json_serialize_data_jrjackson(data)
15
- JrJackson::Json.dump(data)
16
- end
17
-
18
- DATASETS = {
19
- :keys_50 => generate_data_array_for_spec([3, 3, 3, 2]),
20
- :keys_200 => generate_data_array_for_spec([4, 4, 3, 4]),
21
- :keys_200_flat => generate_data_array_for_spec([200]),
22
- :keys_512 => generate_data_array_for_spec([8, 4, 4, 4]),
23
- :keys_960 => generate_data_array_for_spec([12, 5, 4, 4]),
24
- :keys_2700 => generate_data_array_for_spec([14, 8, 6, 4])
25
- }
26
-
27
- def run_benchmark_and_print_results(data, run_benchmark_func)
28
- puts ""
29
- puts "Using %s total keys in a hash" % [Scalyr::Common::Util.flatten(data[0]).count]
30
- puts ""
31
-
32
- result = []
33
- ITERATIONS.times do |i|
34
- result << Benchmark.measure { run_benchmark_func.(data[i]) }
35
- end
36
-
37
- sum = result.inject(nil) { |sum, t| sum.nil? ? sum = t : sum += t }
38
- avg = sum / result.size
39
-
40
- Benchmark.bm(7, "sum:", "avg:") do |b|
41
- [sum, avg]
42
- end
43
- puts ""
44
- end
45
-
46
- puts "Using %s iterations" % [ITERATIONS]
47
- puts ""
48
-
49
- puts "native"
50
- puts "==============================="
51
-
52
- # Around ~50 keys in a hash
53
- data = DATASETS[:keys_50]
54
- run_benchmark_and_print_results(data, method(:json_serialize_data_native))
55
-
56
- # Around ~200 keys in a hash
57
- data = DATASETS[:keys_200]
58
- run_benchmark_and_print_results(data, method(:json_serialize_data_native))
59
-
60
- # Around ~200 keys in a hash (single level)
61
- data = DATASETS[:keys_200_flat]
62
- run_benchmark_and_print_results(data, method(:json_serialize_data_native))
63
-
64
- # Around ~2700 keys in a hash
65
- data = DATASETS[:keys_2700]
66
- run_benchmark_and_print_results(data, method(:json_serialize_data_native))
67
-
68
- puts "jrjackson"
69
- puts "==============================="
70
-
71
- # Around ~50 keys in a hash
72
- data = DATASETS[:keys_50]
73
- run_benchmark_and_print_results(data, method(:json_serialize_data_jrjackson))
74
-
75
- # Around ~200 keys in a hash
76
- data = DATASETS[:keys_200]
77
- run_benchmark_and_print_results(data, method(:json_serialize_data_jrjackson))
78
-
79
- # Around ~200 keys in a hash (single level)
80
- data = DATASETS[:keys_200_flat]
81
- run_benchmark_and_print_results(data, method(:json_serialize_data_jrjackson))
82
-
83
- # Around ~2700 keys in a hash
84
- data = DATASETS[:keys_2700]
85
- run_benchmark_and_print_results(data, method(:json_serialize_data_jrjackson))
@@ -1,48 +0,0 @@
1
- require 'benchmark'
2
- require 'quantile'
3
-
4
- require_relative '../../lib/scalyr/common/util'
5
-
6
- # Micro benchmark which measures how much overhead Quantile.observe adds vs random sampling to see
7
- # where making sampling (e.g. on event level metrics) is desired
8
-
9
- ITERATIONS = 10000
10
-
11
- def run_benchmark_and_print_results(run_benchmark_func)
12
- result = []
13
- ITERATIONS.times do |i|
14
- result << Benchmark.measure { run_benchmark_func.() }
15
- end
16
-
17
- sum = result.inject(nil) { |sum, t| sum.nil? ? sum = t : sum += t }
18
- avg = sum / result.size
19
-
20
- Benchmark.bm(7, "sum:", "avg:") do |b|
21
- [sum, avg]
22
- end
23
- puts ""
24
- end
25
-
26
-
27
- puts "Using %s iterations" % [ITERATIONS]
28
- puts ""
29
-
30
- @value = Quantile::Estimator.new
31
- @prng = Random.new
32
-
33
- def quantile_observe()
34
- @value.observe(5)
35
- end
36
-
37
- def random_sample()
38
- return @prng.rand(0.0..1.0) < 0.5
39
- end
40
-
41
- puts "Quartile.observe()"
42
- puts "==============================="
43
-
44
- run_benchmark_and_print_results(method(:quantile_observe))
45
-
46
- puts "random sample"
47
- puts "==============================="
48
- run_benchmark_and_print_results(method(:random_sample))
@@ -1,107 +0,0 @@
1
- require 'benchmark'
2
- require 'quantile'
3
-
4
- require_relative '../../lib/scalyr/constants'
5
- require_relative '../../lib/scalyr/common/util'
6
- require_relative './util'
7
-
8
- # Micro benchmark which measures how long "set_session_level_serverhost_on_events" takes
9
-
10
- ITERATIONS = 100
11
-
12
- def run_benchmark_and_print_results(data, run_benchmark_func)
13
- puts ""
14
- puts "Using %s total events in a batch" % [data[0].size]
15
- puts ""
16
-
17
- result = []
18
- ITERATIONS.times do |i|
19
- result << Benchmark.measure { run_benchmark_func.(data[i]) }
20
- end
21
-
22
- sum = result.inject(nil) { |sum, t| sum.nil? ? sum = t : sum += t }
23
- avg = sum / result.size
24
-
25
- Benchmark.bm(7, "sum:", "avg:") do |b|
26
- [sum, avg]
27
- end
28
- puts ""
29
- end
30
-
31
- # Generate random event with only single event having special server host attribute set which
32
- # represents a worst case scenario since we need to backfill rest of the events.
33
- def generate_events(count)
34
- result = []
35
-
36
- ITERATIONS.times do |iteration|
37
- events = []
38
-
39
- count.times do |index|
40
- event = generate_hash([2])
41
- event[:attrs] = Hash.new
42
- event[:log] = 1
43
-
44
- if index == count - 1
45
- event[:attrs][EVENT_LEVEL_SERVER_HOST_ATTRIBUTE_NAME] = format("test-host-%s", index)
46
- end
47
-
48
- events << event
49
- end
50
-
51
- raise "Assertion failed" unless events.size == count
52
-
53
- result << events
54
- end
55
-
56
- raise "Assertion failed" unless result.size == ITERATIONS
57
- result
58
- end
59
-
60
- def run_func(events)
61
- # NOTE: This function manipulates events in place
62
- events.each_with_index do |event, index|
63
- if index < events.size - 1
64
- # Last event will have _origServerHost set, but others won't
65
- raise "Assertion failed" unless event[:attrs][EVENT_LEVEL_SERVER_HOST_ATTRIBUTE_NAME].nil?
66
- end
67
- end
68
-
69
- Scalyr::Common::Util.set_session_level_serverhost_on_events("session-server-host-dummy", events, {}, true)
70
-
71
- events.each do |event|
72
- raise "Assertion failed" unless event[:attrs][EVENT_LEVEL_SERVER_HOST_ATTRIBUTE_NAME].nil? == false
73
- end
74
- end
75
-
76
-
77
- puts "Using %s iterations" % [ITERATIONS]
78
- puts ""
79
-
80
- @value = Quantile::Estimator.new
81
-
82
- puts "Util.set_session_level_serverhost_on_events()"
83
- puts "==============================="
84
-
85
- # 100 events in a batch
86
- data = generate_events(100)
87
- run_benchmark_and_print_results(data, method(:run_func))
88
-
89
- # 500 events in a batch
90
- data = generate_events(500)
91
- run_benchmark_and_print_results(data, method(:run_func))
92
-
93
- # 1000 events in a batch
94
- data = generate_events(1000)
95
- run_benchmark_and_print_results(data, method(:run_func))
96
-
97
- # 2000 events in a batch
98
- data = generate_events(2000)
99
- run_benchmark_and_print_results(data, method(:run_func))
100
-
101
- # 3000 events in a batch
102
- data = generate_events(3000)
103
- run_benchmark_and_print_results(data, method(:run_func))
104
-
105
- # 5000 events in a batch
106
- data = generate_events(5000)
107
- run_benchmark_and_print_results(data, method(:run_func))