elastic-stats 0.0.1

data/.gitignore

@@ -0,0 +1,41 @@
+*.gem
+*.rbc
+/.config
+/coverage/
+/InstalledFiles
+/pkg/
+/spec/reports/
+/test/tmp/
+/test/version_tmp/
+/tmp/
+
+## Specific to RubyMotion:
+.dat*
+.repl_history
+build/
+
+## Documentation cache and generated files:
+/.yardoc/
+/_yardoc/
+/doc/
+/rdoc/
+
+## Environment normalisation:
+/.bundle/
+/lib/bundler/man/
+
+# for a library or gem, you might want to ignore these files since the code is
+# intended to run in multiple environments; otherwise, check them in:
+# Gemfile.lock
+# .ruby-version
+# .ruby-gemset
+
+# unless supporting rvm < 1.11.0 or doing something fancy, ignore this:
+.rvmrc
+
+Gemfile.lock
+*.bundle
+*.so
+*.o
+*.a
+mkmf.log

data/.ruby-version

@@ -0,0 +1 @@
+ruby-2.1.2

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in elastic-stats.gemspec
+gemspec

data/Guardfile

@@ -0,0 +1,11 @@
+guard :rspec, cmd: 'rspec' do
+  # watch /lib/ files
+  watch(/^lib\/(.+).rb$/) do |m|
+    "spec/#{m[1]}_spec.rb"
+  end
+
+  # watch /spec/ files
+  watch(/^spec\/(.+).rb$/) do |m|
+    "spec/#{m[1]}.rb"
+  end
+end

data/LICENSE

@@ -0,0 +1,22 @@
+The MIT License (MIT)
+
+Copyright (c) 2015 EagerELK
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2015 Jugrens du Toit
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,29 @@
+# Elastic::Stats
+
+TODO: Write a gem description
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+    gem 'elastic-stats'
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install elastic-stats
+
+## Usage
+
+TODO: Write usage instructions here
+
+## Contributing
+
+1. Fork it ( https://github.com/eagerelk/elastic-stats-ruby/fork )
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create a new Pull Request

data/Rakefile

@@ -0,0 +1,9 @@
+require 'rspec/core/rake_task'
+require 'bundler/gem_tasks'
+
+# Run with `rake spec`
+RSpec::Core::RakeTask.new(:spec) do |task|
+  task.rspec_opts = ['--color', '--format', 'documentation']
+end
+
+task default: :spec

data/elastic-stats.gemspec

@@ -0,0 +1,31 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'elastic/stats/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = 'elastic-stats'
+  spec.version       = Elastic::Stats::VERSION
+  spec.authors       = ['Jugrens du Toit']
+  spec.email         = ['jrgns@jrgns.net']
+  spec.summary    = 'An utility to fetch various statistics from Elasticsearch.'
+  spec.homepage      = 'https://github.com/eagerelk/elastic-stats-ruby'
+  spec.license       = 'MIT'
+
+  spec.files         = `git ls-files -z`.split("\x0")
+  spec.executables   = spec.files.grep(/^bin\//) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(/^(test|spec|features)\//)
+  spec.require_paths = ['lib']
+
+  spec.add_development_dependency 'bundler', '~> 1.6'
+  spec.add_development_dependency 'rspec'
+  spec.add_development_dependency 'guard'
+  spec.add_development_dependency 'guard-rspec'
+  spec.add_development_dependency 'rubocop'
+  spec.add_development_dependency 'rake'
+  spec.add_development_dependency 'webmock'
+  spec.add_development_dependency 'codeclimate-test-reporter'
+  spec.add_runtime_dependency 'hashie'
+  spec.add_runtime_dependency 'statsample'
+  spec.add_runtime_dependency 'elasticsearch'
+end

data/lib/elastic/stats.rb

@@ -0,0 +1,7 @@
+require 'elastic/stats/version'
+
+module Elastic
+  # Utility to calculate stats using Elasticsearch as the backend
+  module Stats
+  end
+end

data/lib/elastic/stats/elastic_client.rb

@@ -0,0 +1,30 @@
+require 'elasticsearch'
+
+module Elastic
+  module Stats
+    # Module to set up and manage the Elasticsearch client
+    module ElasticClient
+      attr_writer :client
+
+      def client
+        @client ||= Elasticsearch::Client.new client_options
+      end
+
+      def client_options
+        @client_options ||= default_options
+      end
+
+      def client_options=(options = {})
+        client_options.update(options)
+      end
+
+      private
+
+      def default_options
+        {
+          url: ENV['ELASTICSEARCH_URL']
+        }
+      end
+    end
+  end
+end

data/lib/elastic/stats/ks.rb

@@ -0,0 +1,101 @@
+require 'elastic/stats/elastic_client'
+require 'hashie'
+require 'statsample'
+
+module Elastic
+  module Stats
+    # Utility to determine the KolmogorovSmirnov difference between to sets of
+    # data fetched from Elasticsearch
+    class KS
+      include ElasticClient
+
+      attr_accessor :logger
+      attr_writer :debug, :query
+      attr_reader :indices, :to, :from, :span, :interval, :field
+
+      MULTIPLIERS = {
+        0.001 => 1.95,
+        0.005 => 1.73,
+        0.01 => 1.63,
+        0.025 => 1.48,
+        0.05 => 1.36,
+        0.10 => 1.22
+      }
+
+      # indices should include all possible indices.
+      def initialize(indices, options = {})
+        @indices  = indices
+
+        options = default_options.update(options)
+
+        @to       = options.delete(:to)
+        @span     = options.delete(:span)
+        @interval = options.delete(:interval)
+        @field    = options.delete(:field)
+        @offset   = options.delete(:offset)
+
+        @indices = [indices]  unless @indices.is_a? Array
+        @to      = @to.to_i   if @to.respond_to?(:to_i)
+        @from    = @to - @span
+      end
+
+      def fetch(confidence = 0.05)
+        current  = range(@from, @to)
+        previous = range(@from - @offset, @to - @offset)
+
+        difference = Statsample::Test::KolmogorovSmirnov.new(
+          current, previous
+        ).d
+
+        comparison = calculate(current, previous, confidence)
+        {
+          confidence: confidence, comparison: comparison,
+          difference: difference, different?: (difference > comparison)
+        }
+      end
+
+      def calculate(current, previous, confidence)
+        MULTIPLIERS[confidence] * Math.sqrt(
+          (
+            (current.count + previous.count).to_f /
+            (current.count * previous.count)
+          )
+        )
+      end
+
+      def range(from, to)
+        Hashie::Mash.new(
+          client.search index: indices.join(','), body: query(from, to)
+        ).aggregations.hits_per_minute.buckets.collect(&:doc_count)
+      end
+
+      def query(from, to)
+        @query = Hashie::Mash.new
+        @query.aggregations!.hits_per_minute!.date_histogram = {
+          field: field, interval: interval, min_doc_count: 0,
+          extended_bounds: {
+            min: (from * 1000),
+            max:   (to * 1000)
+          }
+        }
+        @query
+      end
+
+      def debug?
+        @debug ||= ENV['DEBUG']
+      end
+
+      private
+
+      def default_options
+        {
+          to: Time.new.to_i,
+          span: (60 * 60 * 12),
+          interval: '1h',
+          field: '@timestamp',
+          offset: (60 * 60 * 24 * 7)
+        }
+      end
+    end
+  end
+end

data/lib/elastic/stats/version.rb

@@ -0,0 +1,6 @@
+module Elastic
+  # Utility to calculate stats using Elasticsearch as the backend
+  module Stats
+    VERSION = '0.0.1'
+  end
+end

data/spec/elastic/stats/elastic_client_spec.rb

@@ -0,0 +1,51 @@
+require 'spec_helper'
+require 'elastic/stats/elastic_client'
+
+# Testing class for the Elastic::Stats::ElasticClient module
+class ElasticClientTest
+  include Elastic::Stats::ElasticClient
+end
+
+describe Elastic::Stats::ElasticClient do
+  subject { ElasticClientTest.new }
+  it 'allows the client to be set' do
+    client = Object.new
+    subject.client = client
+
+    expect(subject.client).to be client
+  end
+
+  it 'has sane default options' do
+    expect(subject.client_options).to eq(
+      url: ENV['ELASTICSEARCH_URL']
+    )
+  end
+
+  it 'allows the client options to be retrieved' do
+    expect(subject.client_options).to eq(url: ENV['ELASTICSEARCH_URL'])
+  end
+
+  it 'allows the default client options to be added' do
+    logger = Object.new
+    options  = {
+      debug: true,
+      logger: logger
+    }
+    subject.client_options = options
+
+    options.update(url: ENV['ELASTICSEARCH_URL'])
+    expect(subject.client_options).to eq options
+  end
+
+  it 'allows the default client options to be overriden' do
+    logger = Object.new
+    options  = {
+      debug: true,
+      logger: logger,
+      url: 'http://mytesturl.com:9200/'
+    }
+    subject.client_options = options
+
+    expect(subject.client_options).to eq options
+  end
+end

data/spec/elastic/stats/ks_spec.rb

@@ -0,0 +1,71 @@
+require 'spec_helper'
+require 'elastic/stats/ks'
+require 'webmock'
+
+describe Elastic::Stats::KS do
+  it 'has an array of indices' do
+    ks = Elastic::Stats::KS.new('logstash-2015.12.12')
+    expect(ks.indices).to be_a Array
+  end
+
+  it 'has sane defaults' do
+    ks = Elastic::Stats::KS.new('logstash-2015.12.12')
+    now = Time.new
+    expect(ks.to).to be_within(2).of(now.to_i)
+    expect(ks.from).to eq ks.to - (60 * 60 * 12)
+    expect(ks.interval).to eq '1h'
+    expect(ks.field).to eq '@timestamp'
+  end
+
+  it 'accepts various options' do
+    now = Time.new - 120
+    options = {
+      to: now,
+      span: (60 * 60 * 24),
+      interval: '5m',
+      field: '@mytimefield'
+    }
+    ks = Elastic::Stats::KS.new('logstash-2015.12.12', options)
+    expect(ks.to).to eq now.to_i
+    expect(ks.from).to eq(ks.to.to_i - (60 * 60 * 24))
+    expect(ks.interval).to eq '5m'
+    expect(ks.field).to eq '@mytimefield'
+  end
+
+  context 'fetch' do
+    subject do
+      WebMock.stub_request(:get, 'http://localhost:9200/fake/_search')
+        .to_return(
+          body: fixture('successful_search.json'),
+          headers: { 'Content-Type' => 'text/json' }
+        )
+
+      ks = Elastic::Stats::KS.new('fake')
+      ks.client_options = {
+        url: 'http://localhost:9200',
+        log: false
+      }
+      ks.fetch
+    end
+
+    it 'fetches the KS difference stat' do
+      expect(subject).to have_key :difference
+      expect(subject[:difference]).to eq 0
+    end
+
+    it 'fetches the KS different boolean' do
+      expect(subject).to have_key :different?
+      expect(subject[:different?]).to be false
+    end
+
+    it 'fetches the KS confidence level' do
+      expect(subject).to have_key :confidence
+      expect(subject[:confidence]).to eq 0.05
+    end
+
+    it 'fetches the KS comparison value' do
+      expect(subject).to have_key :comparison
+      expect(subject[:comparison]).to eq 0.9616652224137048
+    end
+  end
+end

data/spec/fixtures/basic_search_request.json

@@ -0,0 +1 @@
+{"aggregations": {"hits_per_minute": {"date_histogram": {"extended_bounds": {"max": 1424785374000,"min": 1424781774000},"field": "@timestamp","interval": "1m","min_doc_count": 0}}},"query": {"filtered": {"filter": {"bool": {"must": [{"range": {"@timestamp": {"from": 1424781774000,"to": 1424785374000}}}]}}}},"size": 0}

data/spec/fixtures/successful_search.json

@@ -0,0 +1 @@
+{"_shards": {"failed": 0,"successful": 5,"total": 5},"aggregations": {"hits_per_minute": {"buckets": [{"doc_count": 0,"key": 1424781600000,"key_as_string": "2015-02-24T12:40:00.000Z"},{"doc_count": 3,"key": 1424781900000,"key_as_string": "2015-02-24T12:45:00.000Z"},{"doc_count": 1,"key": 1424782200000,"key_as_string": "2015-02-24T12:50:00.000Z"},{"doc_count": 0,"key": 1424782500000,"key_as_string": "2015-02-24T12:55:00.000Z"}]}},"hits": {"hits": [],"max_score": 0.0,"total": 4},"timed_out": false,"took": 2}

data/spec/helpers/utility.rb

@@ -0,0 +1,3 @@
+def fixture(name)
+  File.read('spec/fixtures/' + name)
+end

data/spec/spec_helper.rb

@@ -0,0 +1,12 @@
+require 'codeclimate-test-reporter'
+require 'helpers/utility'
+require 'bundler/setup'
+Bundler.setup
+
+CodeClimate::TestReporter.start
+
+RSpec.configure do |config|
+  config.alias_example_to :fit, focus: true
+  config.filter_run focus: true
+  config.run_all_when_everything_filtered = true
+end

metadata

@@ -0,0 +1,247 @@
+--- !ruby/object:Gem::Specification
+name: elastic-stats
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+  prerelease: 
+platform: ruby
+authors:
+- Jugrens du Toit
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2015-03-08 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.6'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.6'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: guard
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: guard-rspec
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rubocop
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: webmock
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: codeclimate-test-reporter
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: hashie
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: statsample
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: elasticsearch
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: 
+email:
+- jrgns@jrgns.net
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- .ruby-version
+- Gemfile
+- Guardfile
+- LICENSE
+- LICENSE.txt
+- README.md
+- Rakefile
+- elastic-stats.gemspec
+- lib/elastic/stats.rb
+- lib/elastic/stats/elastic_client.rb
+- lib/elastic/stats/ks.rb
+- lib/elastic/stats/version.rb
+- spec/elastic/stats/elastic_client_spec.rb
+- spec/elastic/stats/ks_spec.rb
+- spec/fixtures/basic_search_request.json
+- spec/fixtures/successful_search.json
+- spec/helpers/utility.rb
+- spec/spec_helper.rb
+homepage: https://github.com/eagerelk/elastic-stats-ruby
+licenses:
+- MIT
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.23
+signing_key: 
+specification_version: 3
+summary: An utility to fetch various statistics from Elasticsearch.
+test_files:
+- spec/elastic/stats/elastic_client_spec.rb
+- spec/elastic/stats/ks_spec.rb
+- spec/fixtures/basic_search_request.json
+- spec/fixtures/successful_search.json
+- spec/helpers/utility.rb
+- spec/spec_helper.rb