elasticsearch-xpack 0.1.0.pre

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 72d019a7e50f9f6e17e5876243f036ab1162a0f8
+  data.tar.gz: da5a762394b24d1b99f77c0ad7b4e31916b620c6
+SHA512:
+  metadata.gz: 41e5df6ce07833d9a8b979da33e321d2279464f0956a963ec472666a8cee8dab484a30ceb8cafb215084cc51c3b6df9c1bd9682ed3c975db406c37d2e64ed077
+  data.tar.gz: f35e4f346658bd0f2f6e803e29eda9449fe1e0be21870d4b6e4591aacecf0ee94ce0bb38955703cc9ac44fe9dd27212dfb0c242b260d0c5dc3291498fa8b5f81

data/.gitignore

@@ -0,0 +1,9 @@
+/.bundle/
+/.yardoc
+/Gemfile.lock
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/

data/.travis.yml

@@ -0,0 +1,4 @@
+language: ruby
+rvm:
+  - 2.1.5
+before_install: gem install bundler -v 1.10.6

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in elasticsearch-xpack.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2016 Karel Minarik
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,62 @@
+# Elasticsearch::XPack
+
+A Ruby integration for the [X-Pack extensions](https://www.elastic.co/v5)
+for Elasticsearch.
+
+
+## Installation
+
+Install the package from [Rubygems](https://rubygems.org):
+
+    gem install elasticsearch-xpack
+
+To use an unreleased version, either add it to your `Gemfile` for [Bundler](http://gembundler.com):
+
+    gem 'elasticsearch-xpack', git: 'git://github.com/elastic/elasticsearch-ruby-xpack.git'
+
+or install it from a source code checkout:
+
+    git clone https://github.com/elasticsearch/elasticsearch-ruby-xpack.git
+    bundle install
+    rake install
+
+## Usage
+
+If you use the official [Ruby client for Elasticsearch](https://github.com/elastic/elasticsearch-ruby),
+require the library in your code, and all the methods will be automatically available in the `xpack` namespace:
+
+```ruby
+require 'elasticsearch'
+require 'elasticsearch/xpack'
+
+client = Elasticsearch::Client.new url: 'http://elastic:changeme@localhost:9200'
+
+client.xpack.info
+# => {"build"=> ..., "features"=> ...}
+```
+
+The integration is designed as a standalone `Elasticsearch::XPack::API` module, so it's easy
+to mix it into a different client, and the methods will be available in the top namespace.
+
+For documentation, look into the RDoc annotations in the source files, which contain links to the
+official [X-Pack for the Elastic Stack](https://www.elastic.co/guide/en/x-pack/current/index.html) documentation.
+
+For examples, look into the [`examples`](examples) folder in this repository.
+
+## License
+
+This software is licensed under the Apache 2 license, quoted below.
+
+    Copyright (c) 2016 Elasticsearch <http://www.elasticsearch.org>
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.

data/Rakefile

@@ -0,0 +1,95 @@
+require 'bundler/gem_tasks'
+
+require 'rake/testtask'
+
+task :default do
+  exec "rake --tasks"
+end
+
+Rake::TestTask.new('test:unit') do |test|
+  test.libs << 'test'
+  test.test_files = FileList['test/unit/**/*_test.rb']
+  # test.verbose = true
+  # test.warning = true
+end
+
+namespace :test do
+  desc "Run integration tests"
+  task :integration do
+    require 'ansi'
+
+    suites = %w[
+      x-plugins/elasticsearch/x-pack/src/test/resources/rest-api-spec/test
+      x-plugins/elasticsearch/x-pack/graph/src/test/resources/rest-api-spec/test
+      x-plugins/elasticsearch/x-pack/monitoring/src/test/resources/rest-api-spec/test/monitoring
+      x-plugins/elasticsearch/x-pack/security/src/test/resources/rest-api-spec/test
+      x-plugins/elasticsearch/x-pack/watcher/src/test/resources/rest-api-spec/test/xpack/watcher
+      x-plugins/elasticsearch/x-pack/license-plugin/src/test/resources/rest-api-spec/test
+    ]
+
+    # TEMPORARY
+    disabled_suites = %w[
+    ]
+
+    suites = (suites + disabled_suites).select! { |d| d =~ Regexp.new(ENV['SUITE'].gsub(/,/, '|')) } if ENV['SUITE']
+
+    executed_suites = []
+
+    at_exit do
+      errors = executed_suites.any? { |d| d.values.first == 1 }
+      color  = errors ? :red : :green
+
+      if errors
+        puts "----- ".ansi(color) + "ERROR".ansi(color).ansi(:bold) + ('-'*(80-12)).ansi(color)
+      else
+        puts "----- ".ansi(color) + "SUCCESS".ansi(color).ansi(:bold) + ('-'*(80-13)).ansi(color)
+      end
+
+      executed_suites.each do |d|
+        name = d.keys.first.gsub(%r{x-plugins/elasticsearch/x-pack/([^/]+)/.*}, '\1')
+        status = d.values.first == 0 ? 'OK'.ansi(:green) : 'KO'.ansi(:red)
+        puts "#{status} #{name.ansi(:bold)}"
+      end
+
+      puts ('-'*80).ansi(color)
+
+      exit( errors ? 1 : 0 )
+    end
+
+    suites.each do |suite|
+      begin
+        sh <<-COMMAND
+          TEST_REST_API_SPEC=../#{suite} bundle exec ruby -I lib:test test/integration/yaml_test_runner.rb
+        COMMAND
+        executed_suites << { suite => 0 }
+      rescue RuntimeError
+        executed_suites << { suite => 1 }
+      end
+
+      puts '', '-'*80, ''
+    end
+  end
+end
+
+namespace :elasticsearch do
+  desc "Start Elasticsearch node for tests"
+  task :start do
+    require 'elasticsearch/extensions/test/cluster'
+    Elasticsearch::Extensions::Test::Cluster.start(port: ENV.fetch('TEST_CLUSTER_PORT', 9260), nodes: 1, path_logs: '/tmp')
+  end
+
+  desc "Stop Elasticsearch node for tests"
+  task :stop do
+    require 'elasticsearch/extensions/test/cluster'
+    Elasticsearch::Extensions::Test::Cluster.stop(port: ENV.fetch('TEST_CLUSTER_PORT', 9260), nodes: 1)
+  end
+
+  task :status do
+    require 'elasticsearch/extensions/test/cluster'
+    begin
+      Elasticsearch::Extensions::Test::Cluster.__print_cluster_info(ENV.fetch('TEST_CLUSTER_PORT', 9260))
+    rescue Errno::ECONNREFUSED
+      puts "\e[31m[!] Test cluster not running\e[0m"; exit(1)
+    end
+  end
+end

data/elasticsearch-xpack.gemspec

@@ -0,0 +1,39 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'elasticsearch/xpack/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "elasticsearch-xpack"
+  spec.version       = Elasticsearch::XPack::VERSION
+  spec.authors       = ["Karel Minarik"]
+  spec.email         = ["karel@elastic.co"]
+
+  spec.summary       = "Ruby integrations for the X-Pack extensions for Elasticsearch"
+  spec.description   = "Ruby integrations for the X-Pack extensions for Elasticsearch"
+  spec.homepage      = "https://github.com/elastic/elasticsearch-xpack-ruby"
+  spec.license       = "Apache 2"
+
+  spec.files         = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.require_paths = ["lib"]
+
+  spec.add_development_dependency "bundler", "~> 1.10"
+  spec.add_development_dependency "rake", "~> 11.1"
+
+  spec.add_dependency "elasticsearch-api"
+
+  spec.add_development_dependency "elasticsearch"
+  spec.add_development_dependency "elasticsearch-transport"
+  spec.add_development_dependency "elasticsearch-extensions"
+
+  spec.add_development_dependency "activesupport"
+  spec.add_development_dependency "ansi"
+  spec.add_development_dependency "turn"
+  spec.add_development_dependency "minitest"
+  spec.add_development_dependency "minitest-reporters"
+  spec.add_development_dependency "shoulda-context"
+  spec.add_development_dependency "mocha"
+  spec.add_development_dependency "yard"
+  spec.add_development_dependency "pry"
+end

data/examples/watcher/error_500.rb

@@ -0,0 +1,163 @@
+# An example of a complex configuration for Elasticsearch Watcher alerting and notification system.
+#
+# Execute this file from the root of the repository:
+#
+#     bundle exec ruby -I lib ./examples/watcher/error_500.rb
+#
+# The watch searches for `500` errors in a specific index on a periodic basis, and performs three
+# actions when at least 3 errors have been received in the last 5 minutes:
+#
+# 1. indexes the error documents and aggregations returned from search,
+# 2. sends a notification via e-mail, and
+# 3. sends the data to a HTTP API.
+#
+# If you want to test sending the e-mail, you have to configure Watcher:
+# <https://www.elastic.co/guide/en/x-pack/current/actions-email.html#configuring-email>
+#
+# You can run a simple Sinatra based web server to test the webhook action with this script:
+#
+#     $ ruby -r sinatra -r json -e 'post("/") { json = JSON.parse(request.body.read); puts %Q~Received #{json["watch_id"]} with payload: #{json["payload"]}~ }'
+#
+
+require 'elasticsearch'
+require 'elasticsearch/xpack'
+
+client = Elasticsearch::Client.new url: 'http://elastic:changeme@localhost:9260', log: true
+client.transport.logger.formatter = proc do |severity, datetime, progname, msg| "\e[2m#{msg}\e[0m\n" end
+
+# Delete the Watcher and test indices
+#
+['test_errors', 'alerts', '.watcher-history-*'].each do |index|
+  client.indices.delete index: index, ignore: 404
+end
+
+# Print information about the Watcher plugin
+#
+puts "X-Pack #{client.xpack.info['build']['hash']}"
+
+# Register a new watch
+#
+client.xpack.watcher.put_watch id: 'error_500', body: {
+  # Label the watch
+  #
+  metadata: { tags: ['errors'] },
+
+  # Run the watch every 10 seconds
+  #
+  trigger: { schedule: { interval: '10s' } },
+
+  # Search for at least 3 documents matching the condition
+  #
+  condition: {  compare: { 'ctx.payload.hits.total' => { gt: 3 } } },
+
+  # Throttle the watch execution for 30 seconds
+  #
+  throttle_period: '30s',
+
+  # The search request to execute
+  #
+  input: {
+    search: {
+     request: {
+       indices: ['test_errors'],
+       body: {
+         query: {
+           bool: {
+             must: [
+              { match: { status: 500 } } ,
+              { range: { timestamp: { from: '{{ctx.trigger.scheduled_time}}||-5m',
+                                      to:   '{{ctx.trigger.triggered_time}}' } } }
+             ]
+           }
+         },
+         # Return hosts with most errors
+         #
+         aggregations: {
+           hosts: { terms: { field: 'host' } }
+         }
+    }}}
+  },
+
+  # The actions to perform
+  #
+  actions: {
+    send_email:    {
+      transform: {
+        # Transform the data for the template
+        #
+        script: {
+          lang: 'painless',
+          inline: "[ 'total': ctx.payload.hits.total, 'hosts': ctx.payload.aggregations.hosts.buckets.collect(bucket -> [ 'host': bucket.key, 'errors': bucket.doc_count ]), 'errors': ctx.payload.hits.hits.collect(d -> d._source) ]"
+        }
+      },
+      email: { to:        'alerts@example.com',
+               subject:   '[ALERT] {{ctx.watch_id}}',
+               body:      <<-BODY.gsub(/^ {28}/, ''),
+                            Received {{ctx.payload.total}} errors in the last 5 minutes.
+
+                            Hosts:
+
+                            {{#ctx.payload.hosts}}- {{host}} ({{errors}} errors)\n{{/ctx.payload.hosts}}
+
+                            A file with complete data is attached to this message.\n
+                          BODY
+               attachments: { 'data.yml' => { data: { format: 'yaml' } } }
+             }
+    },
+
+    index_payload: {
+      # Transform the data to be stored
+      #
+      transform: {
+        script: {
+          lang: 'painless',
+          inline: "[ 'watch_id': ctx.watch_id, 'payload': ctx.payload ]"
+        }
+      },
+      index: { index: 'alerts', doc_type: 'alert' }
+    },
+
+    ping_webhook: {
+      webhook: {
+        method: 'post',
+        url:    'http://localhost:4567',
+        body:   %q|{"watch_id" : "{{ctx.watch_id}}", "payload" : "{{ctx.payload}}"}| }
+    }
+  }
+}
+
+# Create the index with example documents
+#
+client.indices.create index: 'test_errors', body: {
+  mappings: {
+    d: {
+      properties: {
+        host: { type: 'keyword' }
+      }
+    }
+  }
+}
+
+# Index documents to trigger the watch
+#
+10.times do
+  client.index index: 'test_errors', type: 'd',
+               body: { timestamp: Time.now.utc.iso8601, status: "#{rand(4..5)}00", host: "10.0.0.#{rand(1..3)}" }
+end
+
+# Wait a bit...
+#
+print "Waiting 30 seconds..."
+$i=0; while $i < 30 do
+  sleep(1); print('.'); $i+=1
+end; puts "\n"
+
+# Display information about watch execution
+#
+client.search(index: '.watcher-history-*', q: 'watch_id:error_500', sort: 'trigger_event.triggered_time:asc')['hits']['hits'].each do |r|
+  puts "#{r['_source']['watch_id']} #{r['_source']['state'].upcase} at #{r['_source']['result']['execution_time']}"
+end
+
+# Delete the watch
+#
+client.xpack.watcher.delete_watch id: 'error_500', master_timeout: '30s', force: true

data/lib/elasticsearch/xpack.rb

@@ -0,0 +1,32 @@
+require "elasticsearch/api"
+require "elasticsearch/xpack/version"
+
+Dir[ File.expand_path('../xpack/api/actions/**/*.rb', __FILE__) ].each   { |f| require f }
+Dir[ File.expand_path('../xpack/api/namespace/**/*.rb', __FILE__) ].each { |f| require f }
+
+module Elasticsearch
+  module XPack
+    module API
+      def self.included(base)
+        Elasticsearch::XPack::API.constants.reject {|c| c == :Client }.each do |m|
+          base.__send__ :include, Elasticsearch::XPack::API.const_get(m)
+        end
+      end
+
+      class Client
+        include Elasticsearch::API::Common::Client, Elasticsearch::API::Common::Client::Base
+        include Elasticsearch::XPack::API
+      end
+    end
+  end
+end
+
+module Elasticsearch
+  module Transport
+    class Client
+      def xpack
+        @xpack_client ||= Elasticsearch::XPack::API::Client.new(self)
+      end
+    end
+  end
+end if defined?(Elasticsearch::Transport::Client)