lumbersexual 0.2.2 → 0.3.0

checksums.yaml

@@ -1,15 +1,15 @@
 ---
 !binary "U0hBMQ==":
   metadata.gz: !binary |-
-    NjNlMmYxZDk5Mzg3NGExNWIxYjVmZjQ0YjBiYzU4N2RlZTkxMzM1YQ==
+    MWY2OWU3ZjAyOWE1NTE1NzY0MDhhNWFmYmI0MjA0OTJiZjUwOTVlOA==
   data.tar.gz: !binary |-
-    ODk2MjQ1N2U1ZmNkZGVhZjc3ZGQwMTc0YjQyNjE3YTM1YzU3MWRhNw==
+    MjhlMjY1M2I5OGJjNDY0MDZmNmNjMzE5YjFlZWEwNzMwZWM3YzQzOQ==
 SHA512:
   metadata.gz: !binary |-
-    MGJlMDA4YzUzNmQyNjE1NzE5ZmRhZDBhMzRiNjI5ZmNlNDc0ZTAxZWZkNjU4
-    N2VlYTJhZTZiZjhmNGRlMmY4ZTM2MTFkMGNkMjJmMWNiOGM5YTRjZTI4YmZk
-    OWQxMTVkOTViYWZhY2U3YTVhOGZhMTI5NDFmZTk5ODNkZGYyYmE=
+    YWQwNDYyNzk5YjhjZGVmZjljZWJkNTgzY2RhNTFjZTU4N2EyMzQwZTM3MWQw
+    NGRmOTllYjc2MGE1MzllNjIzY2Q0ZTk4M2E4MDc1YmEzMGZlMTM0NTM4NWU1
+    YzZkNDQ3Y2E3NjFiNTFiZjA4Zjc1M2EyODEzZDJjN2Q0ZGMwOWY=
   data.tar.gz: !binary |-
-    MzQ0NDIwMzhmZGFjZTY4NDk3YTI4M2M2N2YwOWMzNGFkOTM3OThiNTQ0Zjhi
-    MDg5MjI2MzQyMjgwNTE0YWI1NGY5OTA5ZWY3MWQ4NDhkMDc3ZjJhOWI4Yzcy
-    YTNkOTU5MTQ1NWU1OGYzMWJlMzYzOWIxMzQ3YjMzNmIwMWViNmI=
+    YTk1ZGVjYjdlOTZhNTI5YjI1NTk0NzkwNTRhMDM0OTk5YmU5ZTJmNjk4NTY0
+    YTI0YmI4ZTY1YWI4YTdhYTUzZDdlZGI3MWNiMDM2NzFlZWIxY2Q2MjA0OTVi
+    NGRkOTU4NjAwMjRjMzRkYmM3OTU3YjlhNTE5MjVkOTVmNTMyMjg=

data/.travis.yml

@@ -6,7 +6,7 @@ rvm:
 before_install: gem install bundler -v 1.10.6
 deploy:
   provider: rubygems
-  ruby: 'jruby-9.0.5.0'
+  ruby: jruby-9.0.5.0
   api_key:
     secure: qpkX3I/j5aXJvD5pBdiddZ63uCxMsQFeTsWkingDkLsxV/DPAiTn9orwmU3XnYfZfr7tU2z8UztHzqPtIfFmTD/v6FLDqLUzYL8SpfZIuu4BgzbFhQ7fpw2me6a1ISOGz5wkMQ34klZMuPZjX2x9KjrkDh8ZSsZziHi2cpkbj0JLW91ezW+W8yrQvOm/dch0uXMLEOlbxbjPjHr76bFQqvkc2Z0YprxK2j525t3+SEnVN51DiVGimPlUrIQ+mtjGOqbZYCMw4rBex+BdQS9T4as6pG/BT1Tk92MFE/gRMJg9QO/DqwolB3mGNBEYR+3skGktz2UBMSGu6XDFxEOqOgK0joe+ehfiL1Oq2hnp482vHODl06HjYIjVlsXGT7aD7C4TcA2yrRW5hg8UaYoVkIbiZz26GrXX8vEZ4C4HgRruQkyt31cLvhdRPJvStkXzgQQXbwmS6P3lkRE3gxwYyeuzblpbprg2Nolmt9GO//1yQ497YUsY18tK9fOZr8vz/xiCuDMQRuJcheA9C27Z9nJZB2iH4xTNwNl53eFYhWMVYdkw16JYWJ17xMg+VnWUwkA/jODU0GlXEvNHNFRaDA9qyedIk5ainAnzGJpExCsWuKFJvJl1Qn2bD3/neruCIBoCaxQOdBrGd/579ALyjkujRmgZTOiaqRh4DaYHGrg=
   gem: lumbersexual

data/README.md

@@ -1,28 +1,40 @@
 # Lumbersexual [![Build Status](https://travis-ci.org/sampointer/lumbersexual.svg?branch=master)](https://travis-ci.org/sampointer/lumbersexual) [![Gem Version](https://badge.fury.io/rb/lumbersexual.svg)](https://badge.fury.io/rb/lumbersexual)
 
-<img align="right" width="158" height="144" src="https://github.com/sampointer/lumbersexual/etc/assets/lumber-156795_960_720.png" alt="Lumbersexual" />
-This gem generates random-enough syslog entries for the purposes of testing syslog throughput, ELK stacks, aggregated logging infrastructures and log index performance.
+<img align="right" width="158" height="144" src="etc/assets/lumber-156795_960_720.png" alt="Lumbersexual" />
+Benchmarking tool for the purposes of testing syslog throughput, ELK stacks, aggregated logging infrastructures and log index performance.
+
+## Introduction
+Lumbersexual provides both a means of generating load in order to test log aggregation infrastructures, and the means of measuring the latency of log ingestion by that infrastructure under load.
+
+In its default form `lumbersexual` will generate random-enough syslog entries to generate various types of load against a log aggregation infrastructure: volume, breadth, syslog configuration, indexing performance. Run across many nodes it can be used to tune and test syslog ELK-like infrastructures.
+
+It can also be used to measure latency in a manner familiar to those who remember Maakit for MySQL. This can be used both for benchmarking and, by dint of generating telemetry, as a monitoring component.
+
+Together these two modes enable a logging infrastructure to be placed under stress, the ingestion latency measured and then the ongoing latency sampled and alerted upon.
 
 ## Requirements
 
-Whilst `lumbersexual` will run correctly under MRI 2.3.0 the best performance at scale can be obtained by using jruby-9.0.5.0 or later under Java 7. Furthermore throughput is greatest and most accurate with machines with 2 cores or more. By default twice as many threads as cores will be used.
+Whilst `lumbersexual` will run correctly under MRI 2.2.0 and later the best performance at scale can be obtained by using jruby-9.0.5.0 or later under Java 7. Furthermore throughput is greatest and most accurate with machines with 2 cores or more. By default twice as many threads as cores will be used.
 
 A dictionary file is needed from which to generate the randomized messages. Under Debian-derived distributions `apt-get install wamerican; apt-get install dictionaries-common` is not a bad place to start.
 
 ## Usage
+### Load Generation, The Default Mode
+In the default mode Lumbersexual will generate random-enough syslog entries. Without passing additional options you may find the defaults to be extremely aggressive. A real-word example might be:
 
 ```bash
-$ lumbersexual --help
+$ lumbersexual --maxwords 50 --minwords 4 --rate 100 --statsdhost localhost
 ```
 
-## Telemetry
-By supplying the switch `--statsdhost` with a hostname you can turn on statsd metric generation. Lumbersexual will assume it can write to a statsd-like daemon on UDP 8125 and will supply 2 types of telemetry.
+On a 2 core host `lumbersexual` will attempt to generate 200 syslog entries per second and produce statsd telemetry about the distribution across facilities and priorities of that load. Each log entry will be between 4 and 50 random words.
+
+By supplying the switch `--statsdhost` with a hostname statsd metric generation is enabled. Lumbersexual will assume it can write to a statsd-like daemon on UDP 8125 and will supply 2 types of telemetry.
 
 * During a run each thread will increment a counter at the path 
 ```
 lumbersexual.thread.<UUID>.<facility>.<priority>.messages_sent 
 ```
-(where UUID is a randomized string for each thread) each time a message is successfully sent. The facility and priority are reported in their numeric form to save the overhead of a lookup for each write.
+(where `<UUID>` is a randomized string for each thread) each time a message is successfully sent. The facility and priority are reported in their numeric form to save the overhead of a lookup for each write.
 * At the end of a run the following metric paths will be produced:
 ```
 lumbersexual.run.messages_total # gauge
@@ -32,6 +44,14 @@ lumbersexual.run.rate           # gauge
 
 It is up to you to use the aggregation functions of your telemetry system to combine these into a form you find acceptable.
 
+### Ingestion Latency
+Ingestion latency mode generates a syslog message with a unique identifier and then repeatedly queries the supplied ElasticSearch endpoint until that message is returned.  With the addition of the `--statsdhost` switch telemetry about this timing is generated. This is both useful for understanding the performance of a logging infrastructure under load and as a component in a telemetery-based monitoring system.
+```bash
+$ lumbersexual --latency --uri https://my.elasticsearch.cluster:9200 --statsdhost localhost
+```
+
+The `--all` switch can be used to choose between searching today's index only (be careful around midnight!), or across all indices. The latter is useful if you've a rolling online retention period and want to observe the effect on search latency by changes to that.
+
 ## Development
 
 After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment. Run `bundle exec lumbersexual` to use the gem in this directory, ignoring other installed copies of this gem.

data/exe/lumbersexual

@@ -2,17 +2,15 @@
 
 require "lumbersexual"
 require "slop"
-require "syslog"
-require "thread"
-require "timeout"
 require "etc"
-require "statsd-ruby"
-require "securerandom"
+require "uri"
 
 options = Slop.parse do |o|
+  o.bool    '-a', '--all', 'search all indices rather than today\'s in latency mode', default: nil
   o.string  '-D', '--dictionaryfile', 'path to dictionary file (default: /etc/dictionaries-common/words)', default: '/etc/dictionaries-common/words'
   o.array   '-f', '--facilities', 'additional comma-seperated facilities', default: []
   o.bool    '-h', '--help', 'print this message' 
+  o.bool    '-l', '--latency', 'run in latency rather than generation mode (default: no)', default: nil
   o.integer '-M', '--maxwords', 'maximum number of words per message (default: 20)', default: 20
   o.integer '-m', '--minwords', 'minimum number of words per message (default: 3)', default: 3
   o.array   '-p', '--priorities', 'additional comma-seperated priorities', default: []
@@ -20,6 +18,7 @@ options = Slop.parse do |o|
   o.string  '-s', '--statsdhost', 'send statsd telemetry to the named host (default: off)', default: nil
   o.integer '-T', '--threads', 'number of threads (defaults to number of cores * 2)', default: Etc.nprocessors * 2
   o.integer '-t', '--timeout', 'length of execution. 0 for forever (default: 0)', default: 0
+  o.string  '-u', '--uri', 'elasticsearch uri when run in latency mode (default: http://localhost:9200/)', default: URI::parse("http://localhost:9200/")
 end
 
 if options.help?
@@ -27,91 +26,26 @@ if options.help?
   exit
 end
 
+options[:uri] = URI.parse(options[:uri].to_s)
+
 trap('INT') {
   raise Interrupt
 }
 
-# Get our word corpus and define the priorities and facilities we can use
-facilities = [ Syslog::LOG_ALERT, Syslog::LOG_CRIT, Syslog::LOG_ERR, Syslog::LOG_WARNING, Syslog::LOG_NOTICE, Syslog::LOG_INFO ]
-options[:facilities].each { |f| facilities << Object.const_get('Syslog').const_get("LOG_#{f.upcase}") }
-
-priorities = [ Syslog::LOG_AUTHPRIV, Syslog::LOG_CRON, Syslog::LOG_DAEMON, Syslog::LOG_FTP, Syslog::LOG_LPR, Syslog::LOG_MAIL, Syslog::LOG_NEWS, Syslog::LOG_SYSLOG, Syslog::LOG_USER, Syslog::LOG_UUCP]
-0..7.times { |n| priorities << Object.const_get('Syslog').const_get("LOG_LOCAL#{n}")}
-options[:priorities].each { |p| priorities << Object.const_get('Syslog').const_get("LOG_#{p.upcase}") }
-
-words = []
-raise "Unable to find dictionary file at #{options[:dictionaryfile]}" unless File.exist?(options[:dictionaryfile])
-File.open(options[:dictionaryfile]).each_line { |l| words << l.chomp }
-
-case options[:rate]
-when 0
- pause = 0.0
+case options[:latency]
+when true
+  require 'lumbersexual/plugin/latency'
+  plugin = Lumbersexual::Plugin::Latency.new(options)
 else
-  pause = 1.0 / options[:rate]
+  require 'lumbersexual/plugin/load_generator'
+  plugin = Lumbersexual::Plugin::LoadGenerator.new(options)
 end
 
-puts "Runtime: #{RUBY_VERSION} #{RUBY_PLATFORM}"
-puts "Loaded #{words.size} words"
-puts "Timeout: #{options[:timeout]}"
-puts "Threads: #{options[:threads]}"
-puts "Rate per thread: #{options[:rate]}/s"
-puts "Total rate: #{options[:rate] * options[:threads]}/s"
-puts "Minimum words per message: #{options[:minwords]}"
-puts "Maximum words per message: #{options[:maxwords]}"
-puts "Statsd host: #{options[:statsdhost]}" if options[:statsdhost]
-puts "Running ..."
-
-# Run until we're done
-global_count = 0
-threads = []
-mutex = Mutex.new
-statsd_global = Statsd.new if options[:statsdhost]
-start_time = Time.now
-
 begin
-  options[:threads].times do
-    threads << Thread.new {
-      # Configure telemetry
-      statsd = Statsd.new(options[:statsdhost]).tap { |s| s.namespace = "lumbersexual.thread.#{SecureRandom.uuid}" } if options[:statsdhost]
-
-      while true do
-        # Connect to syslog with some sane options and log a message
-        message = String.new
-        number_of_words = rand(options[:minwords]..options[:maxwords])
-        words.sample(number_of_words).each { |w| message << "#{w} " }
-        ident = "lumbersexual-#{words.sample}"
-        facility = facilities.sample
-        priority = priorities.sample
-
-        sleep pause
-        mutex.synchronize {
-          syslog = Syslog.open(ident, Syslog::LOG_CONS | Syslog::LOG_NDELAY | Syslog::LOG_PID, priority)
-          syslog.log(facility, message)
-          global_count += 1
-          statsd.increment [ facility, priority, 'messages_sent' ].join('.') if options[:statsdhost]
-          syslog.close
-        }
-
-      end
-    }
-
-  end
-
-  Timeout::timeout(options[:timeout]) {
-    threads.each {|t| t.join}
-  }
-
+  plugin.perform
 rescue Timeout::Error, Interrupt
-  end_time = Time.now
-  elapsed = end_time - start_time
-  rate = global_count / elapsed
-  statsd_global = Statsd.new(options[:statsdhost]).tap { |s| s.namespace = "lumbersexual.run" } if options[:statsdhost]
-  puts "Sent: #{global_count}" 
-  statsd_global.gauge 'messages_total', global_count if options[:statsdhost]
-  puts "Elapsed time: #{elapsed}"
-  statsd_global.timing 'elapsed', elapsed if options[:statsdhost]
-  puts "Messages per second: #{rate}"
-  statsd_global.gauge 'rate', rate if options[:statsdhost]
-  puts "Complete"
-  exit 0
+  plugin.report
 end
+
+puts "Complete"
+exit 0

data/lib/lumbersexual/plugin/latency.rb

@@ -0,0 +1,58 @@
+#!/usr/bin/env ruby
+
+require "statsd-ruby"
+require "securerandom"
+require "uri"
+require "syslog"
+require "timeout"
+require "elasticsearch"
+
+module Lumbersexual
+  module Plugin
+    class Latency
+      def initialize(options, *args)
+        @options = options
+        @found = false
+      end
+
+      def perform
+        elastic = Elasticsearch::Client.new url: @options[:uri], log: true
+
+        if @options[:all]
+          index_name = '_all'
+        else
+          index_name = Time.now.strftime('logstash-%Y.%m.%d')
+        end
+
+        unique = "PING: #{SecureRandom.uuid}"
+        @start_time = Time.now
+        Timeout::timeout(@options[:timeout]) {
+          syslog = Syslog.open('lumbersexual-ping', Syslog::LOG_CONS | Syslog::LOG_NDELAY | Syslog::LOG_PID, Syslog::LOG_INFO)
+          syslog.log(Syslog::LOG_WARNING, unique)
+          syslog.close
+
+          until @found do
+            result = elastic.search index: index_name, q: unique
+            @found = true if result['hits']['total'] == 1
+          end
+        }
+
+        @end_time = Time.now
+        raise Interrupt
+      end
+
+      def report
+        statsd = Statsd.new(@options[:statsdhost]).tap { |s| s.namespace = "lumbersexual.latency" } if @options[:statsdhost]
+
+        if @found
+          puts "Latency: #{@end_time - @start_time}"
+          statsd.gauge 'runs.successful', 1 if @options[:statsdhost]
+          statsd.gauge 'rtt', @end_time - @start_time if @options[:statsdhost] if @options[:statsdhost]
+        else
+          statsd.gauge 'runs.failed', 1 if @options[:statsdhost]
+          puts "Latency: unknown, message not found"
+        end
+      end
+    end
+  end
+end

data/lib/lumbersexual/plugin/load_generator.rb

@@ -0,0 +1,101 @@
+#!/usr/bin/env ruby
+
+require "syslog"
+require "thread"
+require "timeout"
+require "statsd-ruby"
+require "securerandom"
+require "uri"
+
+module Lumbersexual
+  module Plugin
+    class LoadGenerator
+      def initialize(options, *args)
+        @options = options
+      end
+
+      def perform
+        # Get our word corpus and define the priorities and facilities we can use
+        facilities = [ Syslog::LOG_ALERT, Syslog::LOG_CRIT, Syslog::LOG_ERR, Syslog::LOG_WARNING, Syslog::LOG_NOTICE, Syslog::LOG_INFO ]
+        @options[:facilities].each { |f| facilities << Object.const_get('Syslog').const_get("LOG_#{f.upcase}") }
+
+        priorities = [ Syslog::LOG_AUTHPRIV, Syslog::LOG_CRON, Syslog::LOG_DAEMON, Syslog::LOG_FTP, Syslog::LOG_LPR, Syslog::LOG_MAIL, Syslog::LOG_NEWS, Syslog::LOG_SYSLOG, Syslog::LOG_USER, Syslog::LOG_UUCP]
+        0..7.times { |n| priorities << Object.const_get('Syslog').const_get("LOG_LOCAL#{n}")}
+        @options[:priorities].each { |p| priorities << Object.const_get('Syslog').const_get("LOG_#{p.upcase}") }
+
+        words = []
+        raise "Unable to find dictionary file at #{@options[:dictionaryfile]}" unless File.exist?(@options[:dictionaryfile])
+        File.open(@options[:dictionaryfile]).each_line { |l| words << l.chomp }
+
+        case @options[:rate]
+        when 0
+         pause = 0.0
+        else
+          pause = 1.0 / @options[:rate]
+        end
+
+        puts "Runtime: #{RUBY_VERSION} #{RUBY_PLATFORM}"
+        puts "Loaded #{words.size} words"
+        puts "Timeout: #{@options[:timeout]}"
+        puts "Threads: #{@options[:threads]}"
+        puts "Rate per thread: #{@options[:rate]}/s"
+        puts "Total rate: #{@options[:rate] * @options[:threads]}/s"
+        puts "Minimum words per message: #{@options[:minwords]}"
+        puts "Maximum words per message: #{@options[:maxwords]}"
+        puts "Statsd host: #{@options[:statsdhost]}" if @options[:statsdhost]
+        puts "Running ..."
+
+        # Run until we're done
+        @global_count = 0
+        threads = []
+        mutex = Mutex.new
+        @start_time = Time.now
+
+        @options[:threads].times do
+          threads << Thread.new {
+            # Configure telemetry
+            statsd = Statsd.new(@options[:statsdhost]).tap { |s| s.namespace = "lumbersexual.thread.#{SecureRandom.uuid}" } if @options[:statsdhost]
+
+            while true do
+              # Connect to syslog with some sane @options and log a message
+              message = String.new
+              number_of_words = rand(@options[:minwords]..@options[:maxwords])
+              words.sample(number_of_words).each { |w| message << "#{w} " }
+              ident = "lumbersexual-#{words.sample}"
+              facility = facilities.sample
+              priority = priorities.sample
+
+              sleep pause
+              mutex.synchronize {
+                syslog = Syslog.open(ident, Syslog::LOG_CONS | Syslog::LOG_NDELAY | Syslog::LOG_PID, priority)
+                syslog.log(facility, message)
+                @global_count += 1
+                statsd.increment [ facility, priority, 'messages_sent' ].join('.') if @options[:statsdhost]
+                syslog.close
+              }
+
+            end
+          }
+
+        end
+
+        Timeout::timeout(@options[:timeout]) {
+          threads.each {|t| t.join}
+        }
+      end
+
+      def report
+        end_time = Time.now
+        elapsed = end_time - @start_time
+        rate = @global_count / elapsed
+        statsd_global = Statsd.new(@options[:statsdhost]).tap { |s| s.namespace = "lumbersexual.run" } if @options[:statsdhost]
+        puts "Sent: #{@global_count}" 
+        statsd_global.gauge 'messages_total', @global_count if @options[:statsdhost]
+        puts "Elapsed time: #{elapsed}"
+        statsd_global.timing 'elapsed', elapsed if @options[:statsdhost]
+        puts "Messages per second: #{rate}"
+        statsd_global.gauge 'rate', rate if @options[:statsdhost]
+      end
+    end
+  end
+end

data/lib/lumbersexual/version.rb

@@ -1,3 +1,3 @@
 module Lumbersexual
-  VERSION = "0.2.2"
+  VERSION = "0.3.0"
 end

data/lumbersexual.gemspec

@@ -35,4 +35,5 @@ Gem::Specification.new do |spec|
 
   spec.add_dependency "slop", "~> 4.3.0"
   spec.add_dependency "statsd-ruby", "~> 1.3.0"
+  spec.add_dependency "elasticsearch", "~> 1.0.17"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: lumbersexual
 version: !ruby/object:Gem::Version
-  version: 0.2.2
+  version: 0.3.0
 platform: ruby
 authors:
 - Sam Pointer
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2016-04-16 00:00:00.000000000 Z
+date: 2016-04-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -80,6 +80,20 @@ dependencies:
     - - ~>
       - !ruby/object:Gem::Version
         version: 1.3.0
+- !ruby/object:Gem::Dependency
+  name: elasticsearch
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 1.0.17
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 1.0.17
 description: This gem generates random-enough syslog entries for the purposes of testing
   syslog throughput, ELK stacks, aggregated logging infrastructures and log index
   performance.
@@ -103,6 +117,8 @@ files:
 - etc/assets/lumber-156795_960_720.png
 - exe/lumbersexual
 - lib/lumbersexual.rb
+- lib/lumbersexual/plugin/latency.rb
+- lib/lumbersexual/plugin/load_generator.rb
 - lib/lumbersexual/version.rb
 - lumbersexual.gemspec
 homepage: https://github.com/sampointer/lumbersexual