logstash-output-scalyr 0.2.6 → 0.2.8.beta

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: d304cddbc773ab520aac6f831897fc0ea1e00aef12f28528b31f388891048069
-  data.tar.gz: 970a67aef4939a74f2032f5aeefec859bb29b4fd4ec34955bb47d173427a1331
+  metadata.gz: da91d80d418081307dc1a6b756f17024712c0eab9a6d527a140288976a58b76f
+  data.tar.gz: 2ac792b42939edb371614c73c4dbb50fcc835f199d03685ff619d9d355c6465a
 SHA512:
-  metadata.gz: 81535be5b8b2d5a377c09803e0af45a35e581b42c10b4576155f9482313bdc2a8d43444cf3ee886b825705507ca292f599326046cf4b1e516afb33979ac7de37
-  data.tar.gz: f0c566e5d739121926a5470258572386e0e587865e65499fac2dea040193d655283439faa4ebf72c38905f0a040db5a091cb0a1b45840e74f0360fc2c576a635
+  metadata.gz: 0a87ca7115a2eb3f03282046510bd51869e575eb29b8bec6ee273cbc9f68fea0167890835ffdda8bd0d571ac4299aff689642542d8dd5d88eaa2a89fe760a3f0
+  data.tar.gz: 7457fff5003d10b7288a9a958f3ec3f4a490c19da51a3ac3ade0402a37f74fafffc2501185857b4b3a7263753cb6af3276471724870efa0dca9e0773671b2f3f

data/CHANGELOG.md

@@ -1,5 +1,31 @@
 # Beta
 
+## 0.2.8.beta
+
+* Update ``.gemspec`` gem metadata to not include ``spec/`` directory with the tests and tests
+  fixtures with the actual production gem file.
+
+* Do not retry requests that will never be accepted by the server.
+  Specifically, any request that returns HTTP Status code 413 is too large, and
+  will never be accepted. Instead of simply retrying for 10 minutes before
+  sending the request to the DLQ, skip the retries go directly to sending the
+  request to the DLQ.
+
+  To be notified when an event fails to be ingested for whatever reason, create
+  an alert using the query: ``parser='logstash_plugin_metrics'
+  failed_events_processed > 0``. Instructions on how to create an alert can be
+  found in our docs here: https://scalyr.com/help/alerts
+
+## 0.2.7.beta
+
+* SSL cert validation code has been simplified. Now ``ssl_ca_bundle_path`` config option
+  defaults to the CA bundle path which is vendored / bundled with the RubyGem and includes CA
+  certs of the authorities which are used for DataSet API endpoint certificates.
+
+  In addition to that, ``append_builtin_cert`` config option has been removed and the code now
+  throws error in case invalid / inexistent path is specified for the ``ssl_ca_bundle_path``
+  config option - this represents a fatal config error.
+
 ## 0.2.6.beta, 0.2.6
 
 * Update default value of ``ssl_ca_bundle_path`` config option to

data/README.md

@@ -132,42 +132,23 @@ set on the event object to prevent API from rejecting an invalid request.
 
 ## Note On Server SSL Certificate Validation
 
-By default when validating DataSet endpoint server SSL certificate, logstash plugin will use a
-combination of system CA certs bundle from ``/etc/ssl/certs/ca-certificates.crt`` and combination
-of root CA certificates which are bundled with this plugin which represent root certificates used
-to issue / sign server certificates used by the DataSet API endpoint.
+By default when validating DataSet endpoint server SSL certificate, logstash uses CA certificate
+bundles which is vendored / bundled with the RubyGem / plugin. This bundle includes CA certificate
+files of authoried which are used to issue DataSet API endpoint certificates.
 
-In case you want to use only system CA certs bundle (not use certs which are bundled with the
-plugin), you can do that by using the following config options:
+If you want to use system CA bundle, you should update ``ssl_ca_bundle_path`` to system CA bundle
+path (e.g. ``/etc/ssl/certs/ca-certificates.crt``), as shown in the example below:
 
 ```
 output {
  scalyr {
    api_write_token => 'SCALYR_API_KEY'
    ...
-   # You only need to set this config option in case default CA bundle path on your system is
-   # different
    ssl_ca_bundle_path => "/etc/ssl/certs/ca-certificates.crt"
-   append_builtin_cert => false
  }
 }
 ```
 
-In case you want to use only root CA certs which are bundled with the plugin (not use system CA
-certs bundle), you can do that by using the following config options:
-
-```
-output {
- scalyr {
-   api_write_token => 'SCALYR_API_KEY'
-   ...
-   # You only need to set this config option in case default CA bundle path on your system is
-   # different
-   ssl_ca_bundle_path => nil
-   append_builtin_cert => true
- }
-}
-```
 
 ## Options
 
@@ -185,7 +166,7 @@ output {
 
 - Path to SSL CA bundle file which is used to verify the server certificate.
 
-`config :ssl_ca_bundle_path, :validate => :string, :default => "/etc/ssl/certs/ca-certificates.crt"`
+`config :ssl_ca_bundle_path, :validate => :string, :default => CA_CERTS_PATH`
 
 If for some reason you need to disable server cert validation (you are strongly recommended to
 not disable it unless specifically instructed to do so or have a valid reason for it), you can do
@@ -460,6 +441,12 @@ Or to run a single test function defined on line XXX
 bundle exec rspec spec/scalyr/common/util_spec.rb:XXX
 ```
 
+Or using more verbose output mode:
+
+```bash
+bundle exec rspec -fd spec/scalyr/common/util_spec.rb
+```
+
 ## Instrumentation and metrics
 
 By default, plugin logs a special line with metrics to Scalyr every 5 minutes. This line contains
@@ -518,10 +505,9 @@ To deploy the current code on your machine run these commands:
 
 ```
 rm -rf vendor/
-bundle check --path vendor/bundle || bundle install --deployment
+bundle install
 curl -u RUBY_USER:RUBY_PASSWORD https://rubygems.org/api/v1/api_key.yaml > ~/.gem/credentials
 chmod 0600 ~/.gem/credentials
-bundle exec rake vendor
 bundle exec rspec
 bundle exec rake publish_gem
 ```
@@ -531,10 +517,9 @@ Or as an alternative if ``rake publish_gem`` task doesn't appear to work for wha
 
 ```
 rm -rf vendor/
-bundle check --path vendor/bundle || bundle install --deployment
+bundle install
 curl -u RUBY_USER:RUBY_PASSWORD https://rubygems.org/api/v1/api_key.yaml > ~/.gem/credentials
 chmod 0600 ~/.gem/credentials
-bundle exec rake vendor
 bundle exec rspec
 rvm use jruby
 bundle exec gem build logstash-output-scalyr.gemspec

data/lib/logstash/outputs/scalyr.rb

@@ -133,13 +133,14 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
   # Whether or not to verify the connection to Scalyr, only set to false for debugging.
   config :ssl_verify_peer, :validate => :boolean, :default => true
 
-  # Path to SSL bundle file.
-  # Technically, we could also use Ruby specific cert store + using OpenSSL::X509::DEFAULT_CERT_FILE
-  # here, although that variable stores der and not pem format.
-  config :ssl_ca_bundle_path, :validate => :string, :default => "/etc/ssl/certs/ca-certificates.crt"
+  # Path to SSL bundle file used to validate remote / server SSL certificate. By default, path to
+  # the CA bundled which is vendored / bundled with the RubyGem is used.
+  # If user has a specific reason to change this value (e.g. to a system ca bundle such as 
+  # /etc/ssl/certs/ca-certificates.crt, they can update this option).
+  config :ssl_ca_bundle_path, :validate => :string, :default => CA_CERTS_PATH
 
-  # If we should append our built-in Scalyr cert to the one we find at `ssl_ca_bundle_path`.
-  config :append_builtin_cert, :validate => :boolean, :default => true
+  # Unused since v0.2.7, left here for backward compatibility reasons
+  config :append_builtin_cert, :validate => :boolean, :default => false
 
   config :max_request_buffer, :validate => :number, :default => 5500000  # echee TODO: eliminate?
   config :force_message_encoding, :validate => :string, :default => nil
@@ -261,6 +262,10 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
       end
     end
 
+    if not @append_builtin_cert.nil?
+      @logger.warn "append_builtin_cert config option has been deprecated and is unused in versions 0.2.7 and above"
+    end
+
     @dlq_writer = dlq_enabled? ? execution_context.dlq_writer : nil
 
     @message_encoding = nil
@@ -352,7 +357,7 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
     @running = true
     @client_session = Scalyr::Common::Client::ClientSession.new(
         @logger, @add_events_uri,
-        @compression_type, @compression_level, @ssl_verify_peer, @ssl_ca_bundle_path, @append_builtin_cert,
+        @compression_type, @compression_level, @ssl_verify_peer, @ssl_ca_bundle_path,
         @record_stats_for_status, @flush_quantile_estimates_on_status_send,
         @http_connect_timeout, @http_socket_timeout, @http_request_timeout, @http_pool_max, @http_pool_max_per_route
     )
@@ -378,7 +383,7 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
     # This way we don't need to keep idle long running connection open.
     initial_send_status_client_session = Scalyr::Common::Client::ClientSession.new(
         @logger, @add_events_uri,
-        @compression_type, @compression_level, @ssl_verify_peer, @ssl_ca_bundle_path, @append_builtin_cert,
+        @compression_type, @compression_level, @ssl_verify_peer, @ssl_ca_bundle_path,
         @record_stats_for_status, @flush_quantile_estimates_on_status_send,
         @http_connect_timeout, @http_socket_timeout, @http_request_timeout, @http_pool_max, @http_pool_max_per_route
     )
@@ -495,6 +500,24 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
             result.push(multi_event_request)
           end
 
+        rescue Scalyr::Common::Client::PayloadTooLargeError => e
+          # if the payload is too large, we do not retry. we send to DLQ or drop it.
+          exc_data = {
+              :error_class => e.e_class,
+              :url => e.url.to_s,
+              :message => e.message,
+              :batch_num => batch_num,
+              :total_batches => total_batches,
+              :record_count => multi_event_request[:record_count],
+              :payload_size => multi_event_request[:body].bytesize,
+          }
+          exc_data[:code] = e.code if e.code
+          if defined?(e.body) and e.body
+            exc_data[:body] = Scalyr::Common::Util.truncate(e.body, 512)
+          end
+          exc_data[:payload] = "\tSample payload: #{multi_event_request[:body][0,1024]}..."
+          log_retry_failure(multi_event_request, exc_data, 0, 0)
+          next
         rescue Scalyr::Common::Client::ServerError, Scalyr::Common::Client::ClientError => e
           sleep_interval = sleep_for(sleep_interval)
           exc_sleep += sleep_interval
@@ -629,18 +652,25 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
       @multi_receive_statistics[:total_events_processed] += multi_event_request[:logstash_events].length
       @multi_receive_statistics[:failed_events_processed] += multi_event_request[:logstash_events].length
     end
-    message = "Failed to send #{multi_event_request[:logstash_events].length} events after #{exc_retries} tries."
     sample_events = Array.new
     multi_event_request[:logstash_events][0,5].each {|l_event|
       sample_events << Scalyr::Common::Util.truncate(l_event.to_hash.to_json, 256)
     }
-    @logger.error(message, :error_data => exc_data, :sample_events => sample_events, :retries => exc_retries, :sleep_time => exc_sleep)
+    if exc_data[:code] == 413
+      message = "Failed to send #{multi_event_request[:logstash_events].length} events due to exceeding maximum request size. Not retrying non-retriable request."
+      # For PayloadTooLargeError we already include sample Scalyr payload in exc_data so there is no need
+      # to include redundant sample Logstash event objects
+      @logger.error(message, :error_data => exc_data)
+    else
+      message = "Failed to send #{multi_event_request[:logstash_events].length} events after #{exc_retries} tries."
+      @logger.error(message, :error_data => exc_data, :sample_events => sample_events, :retries => exc_retries, :sleep_time => exc_sleep)
+    end
     if @dlq_writer
       multi_event_request[:logstash_events].each {|l_event|
         @dlq_writer.write(l_event, "#{exc_data[:message]}")
       }
     else
-      @logger.warn("Dead letter queue not configured, dropping #{multi_event_request[:logstash_events].length} events after #{exc_retries} tries.", :sample_events => sample_events)
+      @logger.warn("Dead letter queue not configured, dropping #{multi_event_request[:logstash_events].length} events.", :sample_events => sample_events)
     end
   end
 
@@ -1045,7 +1075,7 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
     serialized_request_size = serialized_body.bytesize
 
     # We give it "buffer" since the splitting code allows for some slack and doesn't take into account top-level non-event attributes
-    if not @estimate_each_event_size and serialized_request_size >= @max_request_buffer + 5000
+    if serialized_request_size >= @max_request_buffer + 5000
       # TODO: If we end up here is estimate config opsion is false, split the request here into multiple ones
       @logger.warn("Serialized request size (#{serialized_request_size}) is larger than max_request_buffer (#{max_request_buffer})!")
     end

data/lib/scalyr/common/client.rb

@@ -27,6 +27,18 @@ end
 # An exception that signifies the Scalyr server received the upload request but dropped it
 #---------------------------------------------------------------------------------------------------------------------
 class RequestDroppedError < ServerError;
+  def initialize(msg=nil, code=nil, url=nil, body=nil, e_class="Scalyr::Common::Client::RequestDroppedError")
+    super(msg, code, url, body, e_class)
+  end
+end
+
+#---------------------------------------------------------------------------------------------------------------------
+# An exception that signifies the Scalyr server received the upload request but dropped it due to it being too large.
+#---------------------------------------------------------------------------------------------------------------------
+class PayloadTooLargeError < ServerError;
+  def initialize(msg=nil, code=nil, url=nil, body=nil, e_class="Scalyr::Common::Client::PayloadTooLargeError")
+    super(msg, code, url, body, e_class)
+  end
 end
 
 #---------------------------------------------------------------------------------------------------------------------
@@ -57,7 +69,7 @@ end
 class ClientSession
 
   def initialize(logger, add_events_uri, compression_type, compression_level,
-                 ssl_verify_peer, ssl_ca_bundle_path, append_builtin_cert,
+                 ssl_verify_peer, ssl_ca_bundle_path,
                  record_stats_for_status, flush_quantile_estimates_on_status_send,
                  connect_timeout, socket_timeout, request_timeout, pool_max, pool_max_per_route)
     @logger = logger
@@ -66,7 +78,6 @@ class ClientSession
     @compression_level = compression_level
     @ssl_verify_peer = ssl_verify_peer
     @ssl_ca_bundle_path = ssl_ca_bundle_path
-    @append_builtin_cert = append_builtin_cert
     @record_stats_for_status = record_stats_for_status
     @flush_quantile_estimates_on_status_send = flush_quantile_estimates_on_status_send
     @connect_timeout = connect_timeout
@@ -75,9 +86,6 @@ class ClientSession
     @pool_max = pool_max
     @pool_max_per_route = pool_max_per_route
 
-    # A cert to use by default to avoid issues caused by the OpenSSL library not validating certs according to standard
-    @cert_string = CA_CERT_STRING
-
     # Request statistics are accumulated across multiple threads and must be accessed through a mutex
     @stats_lock = Mutex.new
     @latency_stats = get_new_latency_stats
@@ -120,41 +128,15 @@ class ClientSession
     # verify peers to prevent potential MITM attacks
     if @ssl_verify_peer
       c[:ssl][:verify] = :strict
+      @logger.info("Using CA bundle from #{@ssl_ca_bundle_path} to validate the server side certificate")
 
-      if not @append_builtin_cert
-        # System CA bundle is used, no need to copy it over and append our bundled CA cert
-        @logger.info("Using CA bundle from #{@ssl_ca_bundle_path} to validate the server side certificate")
-        @ca_cert_path = @ssl_ca_bundle_path
-
-        if not File.file?(@ssl_ca_bundle_path)
-          # TODO: For now we don't throw to keep code backward compatible. In the future in case
-          # file doesn't exist, we should throw instead of write empty CA cert file and pass that 
-          # to Manticore which will eventually fail and throw on cert validation
-          #raise Errno::ENOENT.new("ssl_ca_bundle_path config option to an invalid file path which doesn't exist - #{@ssl_ca_bundle_path}")
-          @ca_cert = Tempfile.new("ca_cert")
-          @ca_cert_path = @ca_cert.path
-        end
-      else
-        @ca_cert = Tempfile.new("ca_cert")
-
-        if File.file?(@ssl_ca_bundle_path)
-          @ca_cert.write(File.read(@ssl_ca_bundle_path))
-          @ca_cert.flush
-        else
-          @logger.warn("CA bundle (#{@ssl_ca_bundle_path}) doesn't exist, using only bundled CA certificates")
-        end
-
-        open(@ca_cert.path, "a") do |f|
-          f.puts @cert_string
-        end
-
-        @ca_cert.flush
-        @ca_cert_path = @ca_cert.path
-
-        @logger.info("Using CA bundle from #{@ssl_ca_bundle_path} combined with bundled certificates to validate the server side certificate (#{@ca_cert_path})")
+      if not File.file?(@ssl_ca_bundle_path)
+        raise Errno::ENOENT.new("Invalid path for ssl_ca_bundle_path config option - file doesn't exist or is not readable")
       end
-      c[:ssl][:ca_file] = @ca_cert_path
+
+      c[:ssl][:ca_file] = @ssl_ca_bundle_path
     else
+      @logger.warn("SSL certificate validation has been disabled. You are strongly encouraged to enable it to prevent possible MITM and similar attacks.")
       c[:ssl][:verify] = :disable
     end
 
@@ -338,15 +320,17 @@ class ClientSession
     end
 
     status = response_hash["status"]
+    code = response.code.to_s.strip.to_i
 
     if status != "success"
-      if status =~ /discardBuffer/
+      if code == 413
+        raise PayloadTooLargeError.new(status, response.code, @add_events_uri, response.body)
+      elsif status =~ /discardBuffer/
         raise RequestDroppedError.new(status, response.code, @add_events_uri, response.body)
       else
         raise ServerError.new(status, response.code, @add_events_uri, response.body)
       end
     else
-      code = response.code.to_s.strip.to_i
       if code < 200 or code > 299
         raise ServerError.new(status, response.code, @add_events_uri, response.body)
       end

data/lib/scalyr/constants.rb

@@ -1,15 +1,14 @@
 # encoding: utf-8
 
-PLUGIN_VERSION = "v0.2.6"
+PLUGIN_VERSION = "v0.2.8.beta"
 
 # Special event level attribute name which can be used for setting event level serverHost attribute
 EVENT_LEVEL_SERVER_HOST_ATTRIBUTE_NAME = '__origServerHost'
 
 # Path to the bundled root CA certs used to sign server cert
-CA_CERT_PATH = File.expand_path(File.join(File.dirname(__FILE__), + "/certs/ca_certs.crt"))
+CA_CERTS_PATH = File.expand_path(File.join(File.dirname(__FILE__), + "/certs/ca_certs.crt"))
 
-# Cert which is append to a copy of "/etc/ssl/certs/ca-bundle.crt" file.
-# This is done for backward compatibility and convenience reasons when "appending_builtin_cert"
-# plugin config option is set to true - eventually we want to default it to false and just rely
-# on system ca bundle by default.
-CA_CERT_STRING = File.read(CA_CERT_PATH)
+# Additional check on import to catch this issue early (in case of a invalid path or similar)
+if not File.file?(CA_CERTS_PATH)
+  raise Errno::ENOENT.new("Invalid path specified for CA_CERTS_PATH module constant (likely a developer error).")
+end

data/logstash-output-scalyr.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name = 'logstash-output-scalyr'
-  s.version         = '0.2.6'
+  s.version         = '0.2.8.beta'
   s.licenses = ['Apache-2.0']
   s.summary = "Scalyr output plugin for Logstash"
   s.description     = "Sends log data collected by Logstash to Scalyr (https://www.scalyr.com)"
@@ -10,7 +10,7 @@ Gem::Specification.new do |s|
   s.require_paths = ["lib"]
 
   # Files
-  s.files = Dir['lib/**/*','spec/**/*','*.gemspec','*.md','CONTRIBUTORS','Gemfile','LICENSE','NOTICE.TXT']
+  s.files = Dir['lib/**/*','*.gemspec','*.md','CONTRIBUTORS','Gemfile','LICENSE','NOTICE.TXT']
   # Tests
   s.test_files = s.files.grep(%r{^(test|spec|features)/})
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-output-scalyr
 version: !ruby/object:Gem::Version
-  version: 0.2.6
+  version: 0.2.8.beta
 platform: ruby
 authors:
 - Edward Chee
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2022-08-01 00:00:00.000000000 Z
+date: 2022-10-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -165,15 +165,6 @@ files:
 - lib/scalyr/common/util.rb
 - lib/scalyr/constants.rb
 - logstash-output-scalyr.gemspec
-- spec/benchmarks/bignum_fixing.rb
-- spec/benchmarks/flattening_and_serialization.rb
-- spec/benchmarks/json_serialization.rb
-- spec/benchmarks/metrics_overhead.rb
-- spec/benchmarks/set_session_level_serverhost_on_events.rb
-- spec/benchmarks/util.rb
-- spec/logstash/outputs/scalyr_integration_spec.rb
-- spec/logstash/outputs/scalyr_spec.rb
-- spec/scalyr/common/util_spec.rb
 homepage: https://www.scalyr.com/help/data-sources#logstash
 licenses:
 - Apache-2.0
@@ -191,22 +182,13 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">="
+  - - ">"
     - !ruby/object:Gem::Version
-      version: '0'
+      version: 1.3.1
 requirements: []
 rubyforge_project:
 rubygems_version: 2.7.10
 signing_key:
 specification_version: 4
 summary: Scalyr output plugin for Logstash
-test_files:
-- spec/benchmarks/bignum_fixing.rb
-- spec/benchmarks/flattening_and_serialization.rb
-- spec/benchmarks/json_serialization.rb
-- spec/benchmarks/metrics_overhead.rb
-- spec/benchmarks/set_session_level_serverhost_on_events.rb
-- spec/benchmarks/util.rb
-- spec/logstash/outputs/scalyr_integration_spec.rb
-- spec/logstash/outputs/scalyr_spec.rb
-- spec/scalyr/common/util_spec.rb
+test_files: []

data/spec/benchmarks/bignum_fixing.rb

@@ -1,87 +0,0 @@
-require 'benchmark'
-require 'quantile'
-
-require_relative '../../lib/scalyr/common/util'
-require_relative './util'
-
-# Micro benchmark which measures how long it takes to find all the Bignums in a record and convert them to strings
-
-ITERATIONS = 500
-
-def rand_bignum()
-  return 200004000020304050300 + rand(999999)
-end
-
-def generate_hash(widths)
-  result = {}
-  if widths.empty?
-    return rand_bignum()
-  else
-    widths[0].times do
-      result[rand_str(9)] = generate_hash(widths[1..widths.length])
-    end
-    return result
-  end
-end
-
-def generate_data_array_for_spec(spec)
-  data = []
-  ITERATIONS.times do
-    data << generate_hash(spec)
-  end
-
-  data
-end
-
-def run_benchmark_and_print_results(data, run_benchmark_func)
-  puts ""
-  puts "Using %s total keys in a hash" % [Scalyr::Common::Util.flatten(data[0]).count]
-  puts ""
-
-  result = []
-  ITERATIONS.times do |i|
-    result << Benchmark.measure { run_benchmark_func.(data[i]) }
-  end
-
-  sum = result.inject(nil) { |sum, t| sum.nil? ? sum = t : sum += t }
-  avg = sum / result.size
-
-  Benchmark.bm(7, "sum:", "avg:") do |b|
-    [sum, avg]
-  end
-  puts ""
-end
-
-
-puts "Using %s iterations" % [ITERATIONS]
-puts ""
-
-@value = Quantile::Estimator.new
-@prng = Random.new
-
-def convert_bignums(record)
-  Scalyr::Common::Util.convert_bignums(record)
-end
-
-puts "Util.convert_bignums()"
-puts "==============================="
-
-# Around ~200 keys in a hash
-data = generate_data_array_for_spec([4, 4, 3, 4])
-run_benchmark_and_print_results(data, method(:convert_bignums))
-
-# Around ~200 keys in a hash (single level)
-data = generate_data_array_for_spec([200])
-run_benchmark_and_print_results(data, method(:convert_bignums))
-
-# Around ~512 keys in a hash
-data = generate_data_array_for_spec([8, 4, 4, 4])
-run_benchmark_and_print_results(data, method(:convert_bignums))
-
-# Around ~960 keys in a hash
-data = generate_data_array_for_spec([12, 5, 4, 4])
-run_benchmark_and_print_results(data, method(:convert_bignums))
-
-# Around ~2700 keys in a hash
-data = generate_data_array_for_spec([14, 8, 6, 4])
-run_benchmark_and_print_results(data, method(:convert_bignums))

data/spec/benchmarks/flattening_and_serialization.rb

@@ -1,100 +0,0 @@
-require 'benchmark'
-require 'json'
-
-require_relative '../../lib/scalyr/common/util'
-require_relative './util'
-
-# NOTE: When using jRuby using multiple iterations with the same dataset doesn't make
-# sense since it will just use JITed version of the code which will be very fast. If we
-# wanted to accurately measure using multiple iterations we would need te different
-# input data for each iteration.
-ITERATIONS = 500
-
-def run_benchmark_and_print_results(data, run_benchmark_func)
-  puts ""
-  puts "Using %s total keys in a hash" % [Scalyr::Common::Util.flatten(data[0]).count]
-  puts ""
-
-  result = []
-  ITERATIONS.times do |i|
-    result << Benchmark.measure { run_benchmark_func.(data[i]) }
-  end
-
-  sum = result.inject(nil) { |sum, t| sum.nil? ? sum = t : sum += t }
-  avg = sum / result.size
-
-  Benchmark.bm(7, "sum:", "avg:") do |b|
-    [sum, avg]
-  end
-  puts ""
-end
-
-def flatten_data_func(data)
-  Scalyr::Common::Util.flatten(data)
-end
-
-def json_serialize_data(data)
-  data.to_json
-end
-
-DATASETS = {
-  :keys_50 => generate_data_array_for_spec([3, 3, 3, 2]),
-  :keys_200 => generate_data_array_for_spec([4, 4, 3, 4]),
-  :keys_200_flat => generate_data_array_for_spec([200]),
-  :keys_512 => generate_data_array_for_spec([8, 4, 4, 4]),
-  :keys_960 => generate_data_array_for_spec([12, 5, 4, 4]),
-  :keys_2700 => generate_data_array_for_spec([14, 8, 6, 4])
-}
-
-puts "Using %s iterations" % [ITERATIONS]
-puts ""
-
-puts "Scalyr::Common::Util.flatten()"
-puts "==============================="
-
-# Around ~50 keys in a hash
-data = DATASETS[:keys_50]
-run_benchmark_and_print_results(data, method(:flatten_data_func))
-
-# Around ~200 keys in a hash
-data = DATASETS[:keys_200]
-run_benchmark_and_print_results(data, method(:flatten_data_func))
-
-# Around ~200 keys in a hash (single level)
-data = DATASETS[:keys_200_flat]
-run_benchmark_and_print_results(data, method(:flatten_data_func))
-
-# Around ~512 keys in a hash
-data = DATASETS[:keys_512]
-run_benchmark_and_print_results(data, method(:flatten_data_func))
-
-# Around ~960 keys in a hash
-data = DATASETS[:keys_960]
-run_benchmark_and_print_results(data, method(:flatten_data_func))
-
-# Around ~2700 keys in a hash
-data = DATASETS[:keys_2700]
-run_benchmark_and_print_results(data, method(:flatten_data_func))
-
-puts "JSON.dumps (hash.to_dict)"
-puts "==============================="
-
-# Around ~200 keys in a hash
-data = generate_data_array_for_spec([4, 4, 3, 4])
-run_benchmark_and_print_results(data, method(:json_serialize_data))
-
-# Around ~200 keys in a hash (single level)
-data = DATASETS[:keys_200_flat]
-run_benchmark_and_print_results(data, method(:json_serialize_data))
-
-# Around ~512 keys in a hash
-data = generate_data_array_for_spec([8, 4, 4, 4])
-run_benchmark_and_print_results(data, method(:json_serialize_data))
-
-# Around ~960 keys in a hash
-data = generate_data_array_for_spec([12, 5, 4, 4])
-run_benchmark_and_print_results(data, method(:json_serialize_data))
-
-# Around ~2700 keys in a hash
-data = generate_data_array_for_spec([14, 8, 6, 4])
-run_benchmark_and_print_results(data, method(:json_serialize_data))