logstash-output-scalyr 0.1.7 → 0.1.8

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 9a0be59b3c2ecaeb444f9fb76e6c97129ab9e1eeaa4a907976e4f6e62a44a582
-  data.tar.gz: 4ddecf13bd1b7800d6d5bad0a271014257159aedfb7eb332ebc8f1c1dd0dd3c5
+  metadata.gz: 67c7ee2ee65ab144dc7eebd1fa6a553ef1fb4d6c89558c023f334d8a728e5f2e
+  data.tar.gz: 48027aa03b3d0d5ccf51db08e0b2f2bed125c2d571a07bd64dac2e853485cd48
 SHA512:
-  metadata.gz: 8ed321b57270cf4bc6fbbc3375f516d08df0592d0476fa71d15af40a7afc1ddb4a97634678b65735101f3c4a17422c0d73e3f40524db06444f8c99cfd3f3a375
-  data.tar.gz: 818943ede0fcad23642366f465c63134500cd70fea625d52313dcc628b3bde3f19aecc6739766690f7d5985c9188af78b5c32e63bcfc4e9a1ca60ee0ceac8743
+  metadata.gz: 0cb175245132cb45a1e243383b544b9ee1483084b10eac4a4e44ce7fc1e149fb7bba083a3e6df399e090a25693d4637117abf0f5f765a63fad3cb8c293337b4d
+  data.tar.gz: bb8fc93d41d6a05f517c87bbc1e2e86ec4ceb939f1d08b017434a43057e54aa2919f6e052efe435c7aacb1f9ccc6e61a9ee05409fcebccc5ac44a07d72d556b6

data/CHANGELOG.md

@@ -1,4 +1,12 @@
 # Beta
+## 0.1.8
+
+- Add additional metrics.
+- Correctly handle flatten_nested_values_duration metric
+- Add support for setting sampling rate for per event level metrics. It defaults to %5 (``0.05``.)
+- Status log line format has been updated so it doesn't include comma between key=value pair to
+  make parser definition a bit simpler.
+
 ## 0.1.7
  - Tracking of new statistics such as `multi_receive` method duration and batch sizes.
  - Addition of percentiles for both existing and new stats.

data/README.md

@@ -10,7 +10,7 @@ You can view documentation for this plugin [on the Scalyr website](https://app.s
 # Quick start
 
 1. Build the gem, run `gem build logstash-output-scalyr.gemspec` 
-2. Install the gem into a Logstash installation, run `/usr/share/logstash/bin/logstash-plugin install logstash-output-scalyr-0.1.7.gem` or follow the latest official instructions on working with plugins from Logstash.
+2. Install the gem into a Logstash installation, run `/usr/share/logstash/bin/logstash-plugin install logstash-output-scalyr-0.1.8.gem` or follow the latest official instructions on working with plugins from Logstash.
 3. Configure the output plugin (e.g. add it to a pipeline .conf)
 4. Restart Logstash 
 
@@ -321,6 +321,49 @@ If you want to run just the unit tests, you can run the command displayed below.
 bundle exec rspec spec/logstash/outputs/scalyr_spec.rb spec/scalyr/common/util_spec.rb
 ```
 
+## Instrumentation and metrics
+
+By default, plugin logs a special line with metrics to Scalyr every 5 minutes. This line contains
+various batch, request and event level metrics.
+
+Example line is shown below:
+
+```bash
+plugin_status: total_requests_sent=6 total_requests_failed=0 total_request_bytes_sent=240586 total_compressed_request_bytes_sent=6396 total_response_bytes_received=222 total_request_latency_secs=1.018 total_serialization_duration_secs=0.024 total_compression_duration_secs=0.046 compression_type=deflate compression_level=6 request_latency_p50=0.048 request_latency_p90=0.104 request_latency_p99=0.104 serialization_duration_secs_p50=0.003 serialization_duration_secs_p90=0.005 serialization_duration_secs_p99=0.005 compression_duration_secs_p50=0.006 compression_duration_secs_p90=0.013 compression_duration_secs_p99=0.013 bytes_sent_p50=40116 bytes_sent_p90=40116 bytes_sent_p99=40116 total_multi_receive_secs=1.404 multi_receive_duration_p50=0.083 multi_receive_duration_p90=0.192 multi_receive_duration_p99=0.192 multi_receive_event_count_p50=100 multi_receive_event_count_p90=100 multi_receive_event_count_p99=100 event_attributes_count_p50=8 event_attributes_count_p90=8 event_attributes_count_p99=8 flatten_values_duration_secs_p50=0 flatten_values_duration_secs_p90=0 flatten_values_duration_secs_p99=0
+```
+
+Those lines also have ``logstash_plugin_metrics`` ``parser`` attribute defined which means you
+can easily parse them using a parser definition similar to the one below.
+
+```javascript
+{
+    patterns: {
+      nonQuoted: "[^\"\n ]+"
+  },
+
+  formats: [
+    {
+      format: ".* $_=identifier$=$_=nonQuoted$", repeat: true
+    }
+  ]
+}
+```
+
+Do keep in mind that this line contains quite a lot of metrics so parsing it may make it more likely
+to hit server side defined per 5 minute period unique number of attributes limit in case you are already
+very close to tke limit.
+
+For various request level metrics we track totals (either counts or duration). Those metrics names start
+with ``total_`` (e.g. ``total_requests_failed``). To be able to derive average per request values you
+can do that by dividing the total value with the value of ``total_requests_sent`` metrics.
+
+Because averages are not all that useful we also track percentiles for various request, batch and
+event level metrics. Those metrics names end with ``_p{percentile}``. For example ``_p50`` represents
+50th percentile and ``_p99`` represents 99th percentile (e.g. ``request_latency_p99``).
+
+If you want to change status reporting interval you can do that by changing the
+``status_report_interval`` config option (in seconds).
+
 # Releasing
 
 ## Updating version

data/lib/logstash/outputs/scalyr.rb

@@ -112,6 +112,16 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
   # Whether or not to count status event uploads in the statistics such as request latency etc.
   config :record_stats_for_status, :validate => :boolean, :default => false
 
+  # Sample rate for event level metrics (flattening time, number of attributes per event, etc,).
+  # It's important to set this in case there are many events coming in per seconds, because
+  # instrumentation does add some overhead. By default, we sample 5% of the events. Keep in
+  # mind that we use simple random based sampling. Maximum possible value is 1 (aka no sampling
+  # - record metrics for every single event).
+  # We use sampling since Quantile.observe() operation is more expensive than simple counter
+  # based metric so we need to ensure recording a metric doesn't add too much overhead.
+  # Based on micro benchmark, random based sampler is about 5x faster than quantile.observe()
+  config :event_metrics_sample_rate, :validate => :number, :default => 0.05
+
   # Parser to attach to status events
   config :status_parser, :validate => :string, :default => "logstash_plugin_metrics"
 
@@ -126,6 +136,12 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
 
   public
   def register
+    @prng = Random.new
+
+    if @event_metrics_sample_rate < 0 or @event_metrics_sample_rate > 1
+      raise LogStash::ConfigurationError, "Minimum possible value for 'event_metrics_sample_rate' is 0 (dont sample any events) and maximum is 1 (sample every event)"
+    end
+
     @node_hostname = Socket.gethostname
 
     if @log_constants and not @log_constants.all? { |x| x.is_a? String }
@@ -191,12 +207,13 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
     @last_status_transmit_time_lock = Mutex.new
     @last_status_transmit_time = nil
     @last_status_ = false
-    
-    # Per-multi-receive statistics
+
+    # Plugin level (either per batch or event level metrics). Other request
+    # level metrics are handled by the HTTP Client class.
     @multi_receive_statistics = {
       :total_multi_receive_secs => 0
     }
-    @multi_receive_metrics = get_new_multi_receive_metrics
+    @plugin_metrics = get_new_metrics
 
     # create a client session for uploading to Scalyr
     @running = true
@@ -215,10 +232,12 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
   end # def register
 
   # Convenience method to create a fresh quantile estimator
-  def get_new_multi_receive_metrics
+  def get_new_metrics
     return {
       :multi_receive_duration_secs => Quantile::Estimator.new,
-      :multi_receive_event_count => Quantile::Estimator.new
+      :multi_receive_event_count => Quantile::Estimator.new,
+      :event_attributes_count => Quantile::Estimator.new,
+      :flatten_values_duration_secs => Quantile::Estimator.new
     }
   end
 
@@ -252,7 +271,7 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
         # For some reason a retry on the multi_receive may result in the request array containing `nil` elements, we
         # ignore these.
         if !multi_event_request.nil?
-          @client_session.post_add_events(multi_event_request[:body], false, multi_event_request[:serialization_duration], multi_event_request[:flatten_nested_values_duration])
+          @client_session.post_add_events(multi_event_request[:body], false, multi_event_request[:serialization_duration])
           sleep_interval = 0
           result.push(multi_event_request)
         end
@@ -306,8 +325,8 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
 
     if records_count > 0
       @multi_receive_statistics[:total_multi_receive_secs] += (Time.now.to_f - start_time)
-      @multi_receive_metrics[:multi_receive_duration_secs].observe(Time.now.to_f - start_time)
-      @multi_receive_metrics[:multi_receive_event_count].observe(records_count)
+      @plugin_metrics[:multi_receive_duration_secs].observe(Time.now.to_f - start_time)
+      @plugin_metrics[:multi_receive_event_count].observe(records_count)
     end
 
     send_status
@@ -352,8 +371,6 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
     logs_ids = Hash.new
     next_log_id = 1
 
-    flatten_nested_values_duration = 0
-
     logstash_events.each {|l_event|
 
       record = l_event.to_hash
@@ -467,16 +484,26 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
         record.delete('tags')
       end
 
-      # flatten record
-      flatten_nested_values_duration = 0
+      # Record per-event level metrics (flatten duration, event attributes count). Doing this for every single
+      # event would be somewhat expensive so we use sampling.
+      should_sample_event_metrics = should_sample?
 
+      # flatten record
       if @flatten_nested_values
         start_time = Time.now.to_f
-        record = Scalyr::Common::Util.flatten(record, delimiter=@flatten_nested_values_delimiter) if @flatten_nested_values
+        record = Scalyr::Common::Util.flatten(record, delimiter=@flatten_nested_values_delimiter)
         end_time = Time.now.to_f
         flatten_nested_values_duration = end_time - start_time
       end
 
+      if should_sample_event_metrics
+        @plugin_metrics[:event_attributes_count].observe(record.count)
+
+        if @flatten_nested_values
+          @plugin_metrics[:flatten_values_duration_secs].observe(flatten_nested_values_duration)
+        end
+      end
+
       # Use LogStash event.timestamp as the 'ts' Scalyr timestamp.  Note that this may be overwritten by input
       # filters so may not necessarily reflect the actual originating timestamp.
       scalyr_event = {
@@ -526,7 +553,7 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
           scalyr_events << scalyr_event
           append_event = false
         end
-        multi_event_request = self.create_multi_event_request(scalyr_events, current_threads, logs, flatten_nested_values_duration)
+        multi_event_request = self.create_multi_event_request(scalyr_events, current_threads, logs)
         multi_event_request_array << multi_event_request
 
         total_bytes = 0
@@ -546,13 +573,12 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
     }
 
     # create a final request with any left over events
-    multi_event_request = self.create_multi_event_request(scalyr_events, current_threads, logs, flatten_nested_values_duration)
+    multi_event_request = self.create_multi_event_request(scalyr_events, current_threads, logs)
     multi_event_request_array << multi_event_request
     multi_event_request_array
   end
 
 
-
   # Helper method that adds a client_timestamp to a batch addEvents request body
   def add_client_timestamp_to_body(body)
     current_time_millis = DateTime.now.strftime('%Q').to_i
@@ -562,11 +588,10 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
   end
 
 
-
   # A request comprises multiple Scalyr Events.  This function creates a request hash for
   # final upload to Scalyr (from an array of events, and an optional hash of current threads)
   # Note: The request body field will be json-encoded.
-  def create_multi_event_request(scalyr_events, current_threads, current_logs, flatten_nested_values_duration = 0)
+  def create_multi_event_request(scalyr_events, current_threads, current_logs)
 
     body = {
       :session => @session_id,
@@ -602,31 +627,43 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
     serialized_body = body.to_json
     end_time = Time.now.to_f
     serialization_duration = end_time - start_time
-    { :body => serialized_body, :record_count => scalyr_events.size, :serialization_duration => serialization_duration,
-      :flatten_nested_values_duration => flatten_nested_values_duration }
+    { :body => serialized_body, :record_count => scalyr_events.size, :serialization_duration => serialization_duration }
 
   end  # def create_multi_event_request
 
 
-
-  def get_batch_stats
+  # Retrieve batch and other event level metric values
+  def get_stats
     current_stats = @multi_receive_statistics.clone
 
-    current_stats[:multi_receive_duration_p50] = @multi_receive_metrics[:multi_receive_duration_secs].query(0.5)
-    current_stats[:multi_receive_duration_p90] = @multi_receive_metrics[:multi_receive_duration_secs].query(0.9)
-    current_stats[:multi_receive_duration_p99] = @multi_receive_metrics[:multi_receive_duration_secs].query(0.99)
-    current_stats[:multi_receive_event_count_p50] = @multi_receive_metrics[:multi_receive_event_count].query(0.5)
-    current_stats[:multi_receive_event_count_p90] = @multi_receive_metrics[:multi_receive_event_count].query(0.9)
-    current_stats[:multi_receive_event_count_p99] = @multi_receive_metrics[:multi_receive_event_count].query(0.99)
+    current_stats[:multi_receive_duration_p50] = @plugin_metrics[:multi_receive_duration_secs].query(0.5)
+    current_stats[:multi_receive_duration_p90] = @plugin_metrics[:multi_receive_duration_secs].query(0.9)
+    current_stats[:multi_receive_duration_p99] = @plugin_metrics[:multi_receive_duration_secs].query(0.99)
+
+    current_stats[:multi_receive_event_count_p50] = @plugin_metrics[:multi_receive_event_count].query(0.5)
+    current_stats[:multi_receive_event_count_p90] = @plugin_metrics[:multi_receive_event_count].query(0.9)
+    current_stats[:multi_receive_event_count_p99] = @plugin_metrics[:multi_receive_event_count].query(0.99)
+
+    current_stats[:event_attributes_count_p50] = @plugin_metrics[:event_attributes_count].query(0.5)
+    current_stats[:event_attributes_count_p90] = @plugin_metrics[:event_attributes_count].query(0.9)
+    current_stats[:event_attributes_count_p99] = @plugin_metrics[:event_attributes_count].query(0.99)
+
+    if @flatten_nested_values
+      # We only return those metrics in case flattening is enabled
+      current_stats[:flatten_values_duration_secs_p50] = @plugin_metrics[:flatten_values_duration_secs].query(0.5)
+      current_stats[:flatten_values_duration_secs_p90] = @plugin_metrics[:flatten_values_duration_secs].query(0.9)
+      current_stats[:flatten_values_duration_secs_p99] = @plugin_metrics[:flatten_values_duration_secs].query(0.99)
+    end
 
     if @flush_quantile_estimates_on_status_send
-      @multi_receive_metrics = get_new_multi_receive_metrics
+      @logger.debug "Recreating / reseting quantile estimator classes for plugin metrics"
+      @plugin_metrics = get_new_metrics
     end
+
     current_stats
   end
 
 
-
   # Sends a status update to Scalyr by posting a log entry under the special logfile of 'logstash_plugin.log'
   # Instead of creating a separate thread, let this method be invoked once at startup and then every 5 minutes
   # at most.  (If no events are received, no status update will be sent even if 5 minutes has elapsed).
@@ -655,14 +692,14 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
       @client_session.get_stats.each do |k, v|
         val = v.instance_of?(Float) ? sprintf("%.3f", v) : v
         val = val.nil? ? 0 : val
-        msg << ', ' if cnt > 0
+        msg << ' ' if cnt > 0
         msg << "#{k.to_s}=#{val}"
         cnt += 1
       end
-      get_batch_stats.each do |k, v|
+      get_stats.each do |k, v|
         val = v.instance_of?(Float) ? sprintf("%.3f", v) : v
         val = val.nil? ? 0 : val
-        msg << ', ' if cnt > 0
+        msg << ' ' if cnt > 0
         msg << "#{k.to_s}=#{val}"
         cnt += 1
       end
@@ -670,12 +707,17 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
       status_event[:attrs]['serverHost'] = @node_hostname
       status_event[:attrs]['parser'] = @status_parser
     end
-    multi_event_request = create_multi_event_request([status_event], nil, nil, nil)
-    @client_session.post_add_events(multi_event_request[:body], true, 0, 0)
+    multi_event_request = create_multi_event_request([status_event], nil, nil)
+    @client_session.post_add_events(multi_event_request[:body], true, 0)
     @last_status_transmit_time = Time.now()
     status_event
   end
 
+  # Returns true if we should sample and record metrics for a specific event based on the sampling
+  # rate and random value
+  def should_sample?
+    return @prng.rand(0.0..1.0) < @event_metrics_sample_rate
+  end
 
 
   # Returns true if it is time to transmit status
@@ -704,12 +746,10 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
   end
 
 
-
   # Helper method to check if the dead-letter queue is enabled
   def dlq_enabled?
     # echee TODO submit to DLQ
     respond_to?(:execution_context) && execution_context.respond_to?(:dlq_writer) &&
         !execution_context.dlq_writer.inner_writer.is_a?(::LogStash::Util::DummyDeadLetterQueueWriter)
   end
-
 end

data/lib/scalyr/common/client.rb

@@ -119,14 +119,10 @@ class ClientSession
         :total_request_latency_secs => 0, #  The total number of secs spent waiting for a responses (so average latency
         # can be calculated by dividing this number by @total_requests_sent).
         # This includes connection establishment time.
-        :total_connections_created => 0, # The total number of HTTP connections successfully created.
         :total_serialization_duration_secs => 0, # The total duration (in seconds) it took to serialize (JSON dumos) all the request bodies.
         # You can calculate avg compression duration by diving this value with total_requests_sent
         :total_compression_duration_secs => 0, # The total duration (in seconds) it took to compress all the request bodies.
         # You can calculate avg compression duration by diving this value with total_requests_sent
-        :total_flatten_values_duration_secs => 0, # The total duration (in seconds) it took to flatten nested record values.
-        # In case flattening is disabled, this value will always be 0. Can infer average per-request value by dividing this
-        # value by total_requests_sent
         :compression_type => @compression_type,
         :compression_level => @compression_level,
     }
@@ -160,14 +156,12 @@ class ClientSession
       # The total number of HTTP connections successfully created.
       :serialization_duration_secs => Quantile::Estimator.new, # The duration (in seconds) it took to serialize (JSON dumos) all the request bodies.
       :compression_duration_secs => Quantile::Estimator.new, # The duration (in seconds) it took to compress all the request bodies.
-      :flatten_values_duration_secs => Quantile::Estimator.new, # The duration (in seconds) it took to flatten nested record values.
-      # In case flattening is disabled, this value will always be 0.
       :request_latency_secs => Quantile::Estimator.new, #  Secs spent waiting for a responses. This includes connection establishment time.
       :bytes_sent => Quantile::Estimator.new  # The number of bytes sent over the network. Batch size with a bit more overhead.
     }
   end
 
-  # Get a clone of current statistics hash
+  # Get a clone of current statistics hash and calculate percentiles
   def get_stats
     current_stats = @stats.clone
 
@@ -177,9 +171,6 @@ class ClientSession
     current_stats[:serialization_duration_secs_p50] = @latency_stats[:serialization_duration_secs].query(0.5)
     current_stats[:serialization_duration_secs_p90] = @latency_stats[:serialization_duration_secs].query(0.9)
     current_stats[:serialization_duration_secs_p99] = @latency_stats[:serialization_duration_secs].query(0.99)
-    current_stats[:flatten_values_duration_secs_p50] = @latency_stats[:flatten_values_duration_secs].query(0.5)
-    current_stats[:flatten_values_duration_secs_p90] = @latency_stats[:flatten_values_duration_secs].query(0.9)
-    current_stats[:flatten_values_duration_secs_p99] = @latency_stats[:flatten_values_duration_secs].query(0.99)
     current_stats[:compression_duration_secs_p50] = @latency_stats[:compression_duration_secs].query(0.5)
     current_stats[:compression_duration_secs_p90] = @latency_stats[:compression_duration_secs].query(0.9)
     current_stats[:compression_duration_secs_p99] = @latency_stats[:compression_duration_secs].query(0.99)
@@ -188,6 +179,7 @@ class ClientSession
     current_stats[:bytes_sent_p99] = @latency_stats[:bytes_sent].query(0.99)
 
     if @flush_quantile_estimates_on_status_send
+      @logger.debug "Recreating / reseting quantile estimator classes for plugin metrics"
       @latency_stats = get_new_latency_stats
     end
     current_stats
@@ -196,7 +188,7 @@ class ClientSession
 
 
   # Upload data to Scalyr. Assumes that the body size complies with Scalyr limits
-  def post_add_events(body, is_status, body_serialization_duration = 0, flatten_nested_values_duration = 0)
+  def post_add_events(body, is_status, body_serialization_duration = 0)
     post, compression_duration = prepare_post_object @add_events_uri.path, body
     fail_count = 1  # putative assume failure
     start_time = Time.now
@@ -238,13 +230,11 @@ class ClientSession
           @stats[:total_compressed_request_bytes_sent] += compressed_bytes_sent
           @stats[:total_response_bytes_received] += bytes_received
           @stats[:total_serialization_duration_secs] += body_serialization_duration
-          @stats[:total_flatten_values_duration_secs] += flatten_nested_values_duration
           @stats[:total_compression_duration_secs] += compression_duration
           end_time = Time.now
           @stats[:total_request_latency_secs] += (end_time - start_time)
           @latency_stats[:request_latency_secs].observe(end_time - start_time)
           @latency_stats[:serialization_duration_secs].observe(body_serialization_duration)
-          @latency_stats[:flatten_values_duration_secs].observe(flatten_nested_values_duration)
           @latency_stats[:compression_duration_secs].observe(compression_duration)
           @latency_stats[:bytes_sent].observe(uncompressed_bytes_sent)
         end
@@ -285,7 +275,7 @@ class ClientSession
 
     post = Net::HTTP::Post.new uri_path
     post.add_field('Content-Type', 'application/json')
-    version = 'output-logstash-scalyr 0.1.7'
+    version = 'output-logstash-scalyr 0.1.8'
     post.add_field('User-Agent', version + ';' + RUBY_VERSION + ';' + RUBY_PLATFORM)
 
     if not encoding.nil?

data/logstash-output-scalyr.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name = 'logstash-output-scalyr'
-  s.version         = '0.1.7'
+  s.version         = '0.1.8'
   s.licenses = ['Apache-2.0']
   s.summary = "Scalyr output plugin for Logstash"
   s.description     = "Sends log data collected by Logstash to Scalyr (https://www.scalyr.com)"

data/spec/benchmarks/flattening_and_serialization.rb

@@ -0,0 +1,125 @@
+require 'benchmark'
+require 'json'
+
+require_relative '../../lib/scalyr/common/util'
+
+# NOTE: When using jRuby using multiple iterations with the same dataset doesn't make
+# sense since it will just use JITed version of the code which will be very fast. If we
+# wanted to accurately measure using multiple iterations we would need te different
+# input data for each iteration.
+ITERATIONS = 500
+
+def rand_str(len)
+  return (0...len).map { (65 + rand(26)).chr }.join
+end
+
+def generate_hash(widths)
+  result = {}
+  if widths.empty?
+    return rand_str(20)
+  else
+    widths[0].times do
+      result[rand_str(9)] = generate_hash(widths[1..widths.length])
+    end
+    return result
+  end
+end
+
+def generate_data_array_for_spec(spec)
+  data = []
+  ITERATIONS.times do
+    data << generate_hash(spec)
+  end
+
+  data
+end
+
+def run_benchmark_and_print_results(data, run_benchmark_func)
+  puts ""
+  puts "Using %s total keys in a hash" % [Scalyr::Common::Util.flatten(data[0]).count]
+  puts ""
+
+  result = []
+  ITERATIONS.times do |i|
+    result << Benchmark.measure { run_benchmark_func.(data[0]) }
+  end
+
+  sum = result.inject(nil) { |sum, t| sum.nil? ? sum = t : sum += t }
+  avg = sum / result.size
+
+  Benchmark.bm(7, "sum:", "avg:") do |b|
+    [sum, avg]
+  end
+  puts ""
+end
+
+def flatten_data_func(data)
+  Scalyr::Common::Util.flatten(data)
+end
+
+def json_serialize_data(data)
+  data.to_json
+end
+
+DATASETS = {
+  :keys_50 => generate_data_array_for_spec([3, 3, 3, 2]),
+  :keys_200 => generate_data_array_for_spec([4, 4, 3, 4]),
+  :keys_200_flat => generate_data_array_for_spec([200]),
+  :keys_512 => generate_data_array_for_spec([8, 4, 4, 4]),
+  :keys_960 => generate_data_array_for_spec([12, 5, 4, 4]),
+  :keys_2700 => generate_data_array_for_spec([14, 8, 6, 4])
+}
+
+
+puts "Using %s iterations" % [ITERATIONS]
+puts ""
+
+puts "Scalyr::Common::Util.flatten()"
+puts "==============================="
+
+# Around ~50 keys in a hash
+data = DATASETS[:keys_50]
+run_benchmark_and_print_results(data, method(:flatten_data_func))
+
+# Around ~200 keys in a hash
+data = DATASETS[:keys_200]
+run_benchmark_and_print_results(data, method(:flatten_data_func))
+
+# Around ~200 keys in a hash (single level)
+data = DATASETS[:keys_200_flat]
+run_benchmark_and_print_results(data, method(:flatten_data_func))
+
+# Around ~512 keys in a hash
+data = DATASETS[:keys_512]
+run_benchmark_and_print_results(data, method(:flatten_data_func))
+
+# Around ~960 keys in a hash
+data = DATASETS[:keys_960]
+run_benchmark_and_print_results(data, method(:flatten_data_func))
+
+# Around ~2700 keys in a hash
+data = DATASETS[:keys_2700]
+run_benchmark_and_print_results(data, method(:flatten_data_func))
+
+puts "JSON.dumps (hash.to_dict)"
+puts "==============================="
+
+# Around ~200 keys in a hash
+data = generate_data_array_for_spec([4, 4, 3, 4])
+run_benchmark_and_print_results(data, method(:json_serialize_data))
+
+# Around ~200 keys in a hash (single level)
+data = DATASETS[:keys_200_flat]
+run_benchmark_and_print_results(data, method(:json_serialize_data))
+
+# Around ~512 keys in a hash
+data = generate_data_array_for_spec([8, 4, 4, 4])
+run_benchmark_and_print_results(data, method(:json_serialize_data))
+
+# Around ~960 keys in a hash
+data = generate_data_array_for_spec([12, 5, 4, 4])
+run_benchmark_and_print_results(data, method(:json_serialize_data))
+
+# Around ~2700 keys in a hash
+data = generate_data_array_for_spec([14, 8, 6, 4])
+run_benchmark_and_print_results(data, method(:json_serialize_data))