elasticsearch-embedded 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: f7265763b028eb234cf4ab3a26648a29822075a2
+  data.tar.gz: 9d4ec60f426bbcd6d268df09e75a3cc107a4ad16
+SHA512:
+  metadata.gz: c002d9517dd909df2e743f3c4197dc7843d2f65ba3eb1e424f38c39fada1149e6f44bd75a2005b9026d2d828b1c4c3597aa5e42248f899e2ed7ca1a955be1244
+  data.tar.gz: 9fba9f55a6b0b1ad49419b3d0be76ebd9dbad5efc22503e09945c65fb060480c35edb8baa058543825ea1099200674b058efd46f94e254f5ea8b7380f3c3488c

data/.gitignore

@@ -0,0 +1,22 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp
+*.bundle
+*.so
+*.o
+*.a
+mkmf.log

data/.rspec

@@ -0,0 +1,2 @@
+--color
+--require spec_helper

data/.travis.yml

@@ -0,0 +1,5 @@
+language: ruby
+rvm:
+  - 1.9.3
+  - 2.0.0
+  - 2.1.0

data/Changelog.md

@@ -0,0 +1,3 @@
+## 0.1.0, released 2014-06-25
+
+* Initial version

data/Gemfile

@@ -0,0 +1,14 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in elasticsearch-embedded.gemspec
+gemspec
+
+# Contain a fix for open-uri
+gem 'fakefs', github: 'defunkt/fakefs'
+
+# used for code coverage reports
+gem 'coveralls', require: false
+
+# Useful for debugging
+gem 'pry'
+gem 'awesome_print'

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2014 Fabio Napoleoni
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,132 @@
+# Elasticsearch::Embedded
+
+[![Gem Version](https://badge.fury.io/rb/elasticsearch-embedded.png)](http://badge.fury.io/rb/elasticsearch-embedded) [![Build Status](https://travis-ci.org/fabn/elasticsearch-embedded.svg?branch=master)](https://travis-ci.org/fabn/elasticsearch-embedded) [![Coverage Status](https://coveralls.io/repos/fabn/elasticsearch-embedded/badge.png)](https://coveralls.io/r/fabn/elasticsearch-embedded)
+
+This gem allows to download and execute elasticsearch (single node or local cluster) within a local folder.
+
+It also provides some utilities for usage in RSpec integration tests.
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+    gem 'elasticsearch-embedded'
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install elasticsearch-embedded
+
+## Usage
+
+### Standalone mode
+
+After gem installation you can run `embedded-elasticsearch` executable, it accepts some options for cluster configuration
+
+```
+$ embedded-elasticsearch -h
+Usage: embedded-elasticsearch [options]
+    -w, --working-dir=WORKING_DIR    Elasticsearch working directory (default: `Dir.tmpdir` or `Rails.root.join("tmp")` within rails applications)
+    -p, --port=PORT                  Port on which to run elasticsearch (default: 9250)
+    -c, --cluster-name=NAME          Cluster name (default: elasticsearch_test)
+    -n, --nodes=NODES                Number of nodes started in the cluster (default: 1)
+        --timeout=TIMEOUT            Timeout when starting the cluster (default: 30)
+    -l, --log-level=LEVEL            Logger verbosity, numbers allowed (1..5) or level names (debug, info, warn, error, fatal)
+    -q, --quiet                      Disable stdout logging
+    -S, --show-es-output             Enable elasticsearch output in stdout
+    -V VERSION                       Elasticsearch version to use (default 1.2.1)
+    -P                               Configure cluster to persist data across restarts
+    -h, --help                       Show this message
+    -v, --version                    Show gem version
+```
+
+In order to start a single node cluster (with in memory indices) just run
+
+```
+$ embedded-elasticsearch -w tmp
+Starting ES 1.2.1 cluster with working directory set to /Users/fabio/work/elasticsearch-embedded/tmp. Process pid is 57245
+Downloading elasticsearch 1.2.1 |                                                    ᗧ| 100% (648 KB/sec) Time: 00:00:34
+Starting 1 Elasticsearch nodes........
+--------------------------------------------------------------------------------
+Cluster:            elasticsearch_test
+Status:             green
+Nodes:              1
+                    + node-1 | version: 1.2.1, pid: 57254, address: inet[/0:0:0:0:0:0:0:0:9250]
+
+# Your cluster is running and listening to port 9250
+```
+
+### Usage with foreman
+
+```
+$ cat Procfile
+elasticsearch: embedded-elasticsearch -w tmp
+$ foreman start
+14:53:51 elasticsearch.1 | started with pid 57524
+14:53:51 elasticsearch.1 | Starting ES 1.2.1 cluster with working directory set to /Users/fabionapoleoni/Desktop/work/RubyMine/elasticsearch-embedded/tmp. Process pid is 57524
+14:53:57 elasticsearch.1 | Starting 1 Elasticsearch nodes........
+14:53:57 elasticsearch.1 | --------------------------------------------------------------------------------
+14:53:57 elasticsearch.1 | Cluster:            elasticsearch_test
+14:53:57 elasticsearch.1 | Status:             green
+14:53:57 elasticsearch.1 | Nodes:              1
+14:53:57 elasticsearch.1 |                     + node-1 | version: 1.2.1, pid: 57528, address: inet[/0:0:0:0:0:0:0:0:9250]
+^CSIGINT received
+14:54:02 system          | sending SIGTERM to all processes
+14:54:02 elasticsearch.1 | exited with code 0%
+```
+
+### With RSpec
+
+```ruby
+# In spec/spec_helper.rb
+require 'elasticsearch-embedded'
+# Activate gem behavior with :elasticsearch tagged specs
+Elasticsearch::Embedded::RSpec.configure
+# Alternatively you could specify for which tags you want gem support
+Elasticsearch::Embedded::RSpec.configure_with :search
+
+# In tagged specs
+describe Something, :elasticsearch do
+
+  before(:each) do
+    # all indices are deleted automatically at beginning of each spec
+    expect(client.indices.get_settings).to be_empty
+  end
+
+  # If elastic search client is defined (i.e. with gem 'elasticsearch' or require 'elasticsearch')
+  it 'should make elastic search client available' do
+    expect(client).to be_an_instance_of(::Elasticsearch::Transport::Client)
+    # Create a document into test cluster
+    client.index index: 'test', type: 'test-type', id: 1, body: {title: 'Test'}
+  end
+
+  # Cluster object is also exposed in a helper
+  it 'should make cluster object available' do
+    expect(cluster).to be_an_instance_of(::Elasticsearch::Embedded::Cluster)
+  end
+
+  # If elastic search client is not defined client return an URI instance with base url for the cluster
+  it 'should return cluster uri when elasticsearch is not defined' do
+    ::Elasticsearch.send(:remove_const, :Client)
+    expect(client).to be_an_instance_of(::URI::HTTP)
+    expect(Net::HTTP.get(client)).to include('You Know, for Search')
+  end
+
+end
+
+```
+
+### With Test::Unit/minitest
+
+Pull requests are welcome.
+
+## Contributing
+
+1. Fork it ( https://github.com/fabn/elasticsearch-embedded/fork )
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create a new Pull Request

data/Rakefile

@@ -0,0 +1,7 @@
+require 'bundler/gem_tasks'
+
+require 'rspec/core/rake_task'
+
+RSpec::Core::RakeTask.new(:spec)
+
+task default: :spec

data/bin/embedded-elasticsearch

@@ -0,0 +1,66 @@
+#!/usr/bin/env ruby
+
+require 'elasticsearch-embedded'
+require 'optparse'
+
+cluster = Elasticsearch::Embedded::Cluster.new
+
+OptionParser.new do |opts|
+
+  opts.on '-w', '--working-dir=WORKING_DIR', 'Elasticsearch working directory (default: `Dir.tmpdir` or `Rails.root.join("tmp")` within rails applications)' do |wd|
+    cluster.working_dir = File.expand_path(wd)
+  end
+
+  opts.on '-p', '--port=PORT', Integer, 'Port on which to run elasticsearch (default: 9250)' do |p|
+    cluster.port = p
+  end
+
+  opts.on '-c', '--cluster-name=NAME', 'Cluster name (default: elasticsearch_test)' do |cn|
+    cluster.cluster_name = cn
+  end
+
+  opts.on '-n', '--nodes=NODES', Integer, 'Number of nodes started in the cluster (default: 1)' do |n|
+    cluster.nodes = n
+  end
+
+  opts.on '--timeout=TIMEOUT', Integer, 'Timeout when starting the cluster (default: 30)' do |t|
+    cluster.timeout = t
+  end
+
+  opts.on '-l', '--log-level=LEVEL', "Logger verbosity, numbers allowed (1..5) or level names (#{Logging::LEVELS.keys.join(', ')})" do |l|
+    ::Elasticsearch::Embedded.verbosity(l)
+  end
+
+  opts.on '-q', '--quiet', 'Disable stdout logging' do
+    ::Elasticsearch::Embedded.mute!
+  end
+
+  opts.on '-S', '--show-es-output', 'Enable elasticsearch output in stdout' do
+    cluster.verbose = true
+  end
+
+  opts.on '-V VERSION', "Elasticsearch version to use (default #{Elasticsearch::Embedded::Downloader::DEFAULT_VERSION})" do |v|
+    cluster.version = v
+  end
+
+  opts.on '-P', 'Configure cluster to persist data across restarts' do |p|
+    cluster.persistent = !!p
+  end
+
+  opts.on_tail '-h', '--help', 'Show this message' do
+    puts opts
+    exit
+  end
+
+  opts.on_tail '-v', '--version', 'Show gem version' do
+    puts "Gem version: #{Elasticsearch::Embedded::VERSION}"
+    puts "Elasticsearch version: #{cluster.version}"
+    exit
+  end
+
+end.parse!
+
+# Forward additional arguments to elasticsearch
+cluster.additional_options = ARGV.join(' ') unless ARGV.empty?
+# Start the cluster
+cluster.start_and_wait!

data/elasticsearch-embedded.gemspec

@@ -0,0 +1,31 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'elasticsearch/embedded/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = 'elasticsearch-embedded'
+  spec.version       = Elasticsearch::Embedded::VERSION
+  spec.authors       = ['Fabio Napoleoni']
+  spec.email         = ['f.napoleoni@gmail.com']
+  spec.summary       = %q{Install an embedded version of elasticsearch into your project}
+  spec.homepage      = 'https://github.com/fabn/elasticsearch-embedded'
+  spec.license       = 'MIT'
+
+  spec.files         = `git ls-files -z`.split("\x0")
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ['lib']
+  
+  spec.required_ruby_version = '>= 1.9.2'
+
+  spec.add_runtime_dependency 'ruby-progressbar', '~> 1.5.1'
+  spec.add_runtime_dependency 'rubyzip', '~> 1.0.0'
+  spec.add_runtime_dependency 'logging', '~> 1.8.0'
+
+  spec.add_development_dependency 'bundler', '~> 1.6'
+  spec.add_development_dependency 'rake'
+  spec.add_development_dependency 'rspec', '~> 3.0.0'
+  spec.add_development_dependency 'fakefs', '~> 0.5.0'
+  spec.add_development_dependency 'elasticsearch', '~> 1.0.2'
+end

data/lib/elasticsearch-embedded.rb

@@ -0,0 +1 @@
+require 'elasticsearch/embedded'

data/lib/elasticsearch/embedded.rb

@@ -0,0 +1,15 @@
+require 'elasticsearch/embedded/version'
+require 'elasticsearch/embedded/logger_configuration'
+
+module Elasticsearch
+  module Embedded
+
+    autoload :Downloader, 'elasticsearch/embedded/downloader'
+    autoload :Cluster, 'elasticsearch/embedded/cluster'
+    autoload :RSpec, 'elasticsearch/embedded/rspec_configuration'
+
+    # Configure logging for this module
+    extend LoggerConfiguration
+
+  end
+end

data/lib/elasticsearch/embedded/cluster.rb

@@ -0,0 +1,303 @@
+require 'timeout'
+require 'net/http'
+require 'uri'
+require 'json'
+
+module Elasticsearch
+  module Embedded
+
+    # Class used to manage a local cluster of elasticsearch nodes
+    class Cluster
+
+      # Make logger method available
+      include Logging.globally
+
+      # Options for cluster
+      attr_accessor :port, :cluster_name, :nodes, :timeout, :persistent, :additional_options, :verbose
+
+      # Options for downloader
+      attr_accessor :downloader, :version, :working_dir
+
+      # Assign default values to options
+      def initialize
+        @nodes = 1
+        @port = 9250
+        @version = Downloader::DEFAULT_VERSION
+        @working_dir = Downloader::TEMPORARY_PATH
+        @timeout = 30
+        @cluster_name = 'elasticsearch_test'
+        @pids = []
+        @pids_lock = Mutex.new
+      end
+
+      # Start an elasticsearch cluster and return immediately
+      def start
+        @downloader = Downloader.download(version: version, path: working_dir)
+        start_cluster
+        apply_development_template! if persistent
+      end
+
+      # Start an elasticsearch cluster and wait until running, also register
+      # a signal handler to close the cluster on INT, TERM and QUIT signals
+      def start_and_wait!
+        # register handler before starting cluster
+        register_shutdown_handler
+        # start the cluster
+        start
+        # Wait for all child processes to end then return
+        Process.waitall
+      end
+
+      # Stop the cluster and return after all child processes are dead
+      def stop
+        logger.warn 'Cluster is still starting, wait until startup is complete before sending shutdown command' if @pids_lock.locked?
+        @pids_lock.synchronize do
+          http_object.post('/_shutdown', nil)
+          logger.debug 'Cluster stopped succesfully using shutdown api'
+          Timeout.timeout(2) { Process.waitall }
+          # Reset running pids reader
+          @pids = []
+        end
+      rescue
+        logger.warn "Following processes are still alive #{pids}, kill them with signals"
+        # Send term signal if post request fails to all processes still alive after 2 seconds
+        pids.each { |pid| wait_or_kill(pid) }
+      end
+
+      # Thread safe access to all spawned process pids
+      def pids
+        @pids_lock.synchronize { @pids }
+      end
+
+      # Start server unless it's running
+      def ensure_started!
+        start unless running?
+      end
+
+      # Returns true when started cluster is running
+      #
+      # @return Boolean
+      def running?
+        cluster_health = Timeout::timeout(0.25) { __get_cluster_health } rescue nil
+        # Response is present, cluster name is the same and number of nodes is the same
+        !!cluster_health && cluster_health['cluster_name'] == cluster_name && cluster_health['number_of_nodes'] == nodes
+      end
+
+      # Remove all indices in the cluster
+      #
+      # @return [Array<Net::HTTPResponse>] raw http responses
+      def delete_all_indices!
+        delete_index! :_all
+      end
+
+      # Remove the indices given as args
+      #
+      # @param [Array<String,Symbol>] args list of indices to delet
+      # @return [Array<Net::HTTPResponse>] raw http responses
+      def delete_index!(*args)
+        args.map { |index| http_object.request(Net::HTTP::Delete.new("/#{index}")) }
+      end
+
+      # Used for persistent clusters, otherwise cluster won't get green state because of missing replicas
+      def apply_development_template!
+        development_settings = {
+            template: '*',
+            settings: {
+                number_of_shards: 1,
+                number_of_replicas: 0,
+            }
+        }
+        # Create the template on cluster
+        http_object.put('/_template/development_template', JSON.dump(development_settings))
+      end
+
+      private
+
+      # Build command line to launch an instance
+      def build_command_line(instance_number)
+        [
+            downloader.executable,
+            '-D es.foreground=yes',
+            "-D es.cluster.name=#{cluster_name}",
+            "-D es.node.name=node-#{instance_number}",
+            "-D es.http.port=#{port + (instance_number - 1)}",
+            "-D es.gateway.type=#{cluster_options[:gateway_type]}",
+            "-D es.index.store.type=#{cluster_options[:index_store]}",
+            "-D es.path.data=#{cluster_options[:path_data]}-#{instance_number}",
+            "-D es.path.work=#{cluster_options[:path_work]}-#{instance_number}",
+            '-D es.network.host=0.0.0.0',
+            '-D es.discovery.zen.ping.multicast.enabled=true',
+            '-D es.script.disable_dynamic=false',
+            '-D es.node.test=true',
+            '-D es.node.bench=true',
+            additional_options,
+            verbose ? nil : '> /dev/null'
+        ].compact.join(' ')
+      end
+
+      # Spawn an elasticsearch process and return its pid
+      def launch_instance(instance_number = 1)
+        # Start the process within a new process group to avoid signal propagation
+        Process.spawn(build_command_line(instance_number), pgroup: true).tap do |pid|
+          logger.debug "Launched elasticsearch process with pid #{pid}, detaching it"
+          Process.detach pid
+        end
+      end
+
+      # Return running instances pids, borrowed from code in Elasticsearch::Extensions::Test::Cluster.
+      # This method returns elasticsearch nodes pids and not spawned command pids, they are different because of
+      # elasticsearch shell wrapper used to launch the daemon
+      def nodes_pids
+        # Try to fetch node info from running cluster
+        nodes = JSON.parse(http_object.get('/_nodes/?process').body) rescue []
+        # Fetch pids from returned data
+        nodes.empty? ? nodes : nodes['nodes'].map { |_, info| info['process']['id'] }
+      end
+
+      def start_cluster
+        logger.info "Starting ES #{version} cluster with working directory set to #{working_dir}. Process pid is #{$$}"
+        if running?
+          logger.warn "Elasticsearch cluster already running on port #{port}"
+          wait_for_green(timeout)
+          return
+        end
+        # Launch single node instances of elasticsearch with synchronization
+        @pids_lock.synchronize do
+          1.upto(nodes).each do |i|
+            @pids << launch_instance(i)
+          end
+          # Wait for cluster green state before releasing lock
+          wait_for_green(timeout)
+          # Add started nodes pids to pid array
+          @pids.concat(nodes_pids)
+        end
+      end
+
+      def wait_or_kill(pid)
+        begin
+          Timeout::timeout(2) do
+            Process.kill(:TERM, pid)
+            logger.debug "Sent SIGTERM to process #{pid}"
+            Process.waitpid(pid)
+            logger.info "Process #{pid} exited successfully"
+          end
+        rescue Errno::ESRCH, Errno::ECHILD
+          # No such process or no child => process is already dead
+          logger.debug "Process with pid #{pid} is already dead"
+        rescue Timeout::Error
+          logger.info "Process #{pid} still running after 2 seconds, sending SIGKILL to it"
+          Process.kill(:KILL, pid) rescue nil
+        ensure
+          logger.debug "Removing #{pid} from running pids"
+          @pids_lock.synchronize { @pids.delete(pid) }
+        end
+      end
+
+      # Used as arguments for building command line to launch elasticsearch
+      def cluster_options
+        {
+            port: port,
+            nodes: nodes,
+            cluster_name: cluster_name,
+            timeout: timeout,
+            # command to run is taken from downloader object
+            command: downloader.executable,
+            # persistency options
+            gateway_type: persistent ? 'local' : 'none',
+            index_store: persistent ? 'mmapfs' : 'memory',
+            path_data: File.join(persistent ? downloader.working_dir : Dir.tmpdir, 'cluster_data'),
+            path_work: File.join(persistent ? downloader.working_dir : Dir.tmpdir, 'cluster_workdir'),
+        }
+      end
+
+      # Return an http object to make requests
+      def http_object
+        @http ||= Net::HTTP.new('localhost', port)
+      end
+
+      # Register a shutdown proc which handles INT, TERM and QUIT signals
+      def register_shutdown_handler
+        stopper = ->(sig) do
+          Thread.new do
+            logger.info "Received SIG#{Signal.signame(sig)}, quitting"
+            stop
+          end
+        end
+        # Stop cluster on Ctrl+C, TERM (foreman) or QUIT (other)
+        [:TERM, :INT, :QUIT].each { |sig| Signal.trap(sig, &stopper) }
+      end
+
+      # Waits until the cluster is green and prints information
+      #
+      # @example Print the information about the default cluster
+      #     Elasticsearch::Extensions::Test::Cluster.wait_for_green
+      #
+      # @param (see #__wait_for_status)
+      #
+      # @return Boolean
+      #
+      def wait_for_green(timeout = 60)
+        __wait_for_status('green', timeout)
+      end
+
+      # Blocks the process and waits for the cluster to be in a "green" state.
+      #
+      # Prints information about the cluster on STDOUT if the cluster is available.
+      #
+      # @param status  [String]  The status to wait for (yellow, green)
+      # @param timeout [Integer] The explicit timeout for the operation
+      #
+      # @api private
+      #
+      # @return Boolean
+      #
+      def __wait_for_status(status='green', timeout = 30)
+        Timeout::timeout(timeout) do
+          loop do
+            response = JSON.parse(http_object.get("/_cluster/health?wait_for_status=#{status}").body) rescue {}
+
+            # check response and return if ok
+            if response['status'] == status && nodes == response['number_of_nodes'].to_i
+              __print_cluster_info and break
+            end
+
+            logger.debug "Still waiting for #{status} status in #{cluster_name}"
+            sleep 1
+          end
+        end
+
+        true
+      end
+
+      # Print information about the cluster on STDOUT
+      #
+      # @api private
+      #
+      def __print_cluster_info
+        health = JSON.parse(http_object.get('/_cluster/health').body)
+        nodes = JSON.parse(http_object.get('/_nodes/process,http').body)
+        master = JSON.parse(http_object.get('/_cluster/state').body)['master_node']
+
+        logger.info '-'*80
+        logger.info 'Cluster: '.ljust(12) + health['cluster_name'].to_s
+        logger.info 'Status:  '.ljust(12) + health['status'].to_s
+        logger.info 'Nodes:   '.ljust(12) + health['number_of_nodes'].to_s
+
+        nodes['nodes'].each do |id, info|
+          m = id == master ? '+' : '-'
+          logger.info ''.ljust(12) + "#{m} #{info['name']} | version: #{info['version']}, pid: #{info['process']['id']}, address: #{info['http']['bound_address']}"
+        end
+      end
+
+      # Tries to load cluster health information
+      #
+      # @api private
+      #
+      def __get_cluster_health
+        JSON.parse(http_object.get('/_cluster/health').body) rescue nil
+      end
+
+    end
+
+  end
+end