elasticsearch-embedded 0.1.0

data/lib/elasticsearch/embedded/downloader.rb

@@ -0,0 +1,104 @@
+# coding: utf-8
+require 'tmpdir'
+require 'open-uri'
+require 'ruby-progressbar'
+require 'zip'
+
+module Elasticsearch
+  module Embedded
+    class Downloader
+
+      # Default temporary path used by downloader
+      TEMPORARY_PATH = defined?(Rails) ? Rails.root.join('tmp') : Dir.tmpdir
+      # Default version of elasticsearch to download
+      DEFAULT_VERSION = '1.2.1'
+
+      attr_reader :version, :path
+
+      def initialize(args = {})
+        @version = args[:version] || ENV['ELASTICSEARCH_VERSION'] || DEFAULT_VERSION
+        @path = args[:path] || ENV['ELASTICSEARCH_DOWNLOAD_PATH'] || TEMPORARY_PATH
+      end
+
+      # Download elasticsearch distribution and unzip it in the specified temporary path
+      def self.download(arguments = {})
+        new(arguments).perform
+      end
+
+      def perform
+        download_file
+        extract_file
+        self
+      end
+
+      def downloaded?
+        File.exists?(final_path)
+      end
+
+      def extracted?
+        File.directory?(dist_folder)
+      end
+
+      def dist_folder
+        @dist_folder ||= final_path.gsub /\.zip\Z/, ''
+      end
+
+      def final_path
+        @final_path ||= File.join(working_dir, "elasticsearch-#{version}.zip")
+      end
+
+      def working_dir
+        @working_dir ||= File.realpath(path)
+      end
+
+      def executable
+        @executable ||= File.join(dist_folder, 'bin', 'elasticsearch')
+      end
+
+      private
+
+      def download_file
+        return if downloaded?
+        open(final_path, 'wb') do |target|
+          download_options = {
+              content_length_proc: ->(t) { build_progress_bar(t) },
+              progress_proc: ->(s) { increment_progress(s) }
+          }
+          # direct call here to avoid spec issues with Kernel#open
+          distfile = OpenURI.open_uri("https://download.elasticsearch.org/elasticsearch/elasticsearch/elasticsearch-#{version}.zip", download_options)
+          target << distfile.read
+        end
+      end
+
+      def extract_file
+        return if extracted?
+        Dir.chdir(working_dir) do
+          # Extract archive in path, after block CWD is restored
+          Zip::File.open(final_path) do |zip_file|
+            # Extract all entries into working dir
+            zip_file.each(&:extract)
+          end
+        end
+        # ensure main executable has execute permission
+        File.chmod(0755, executable)
+        # Create folder for log files
+        FileUtils.mkdir(File.join(dist_folder, 'logs'))
+      end
+
+      # Build a progress bar to download elasticsearch
+      def build_progress_bar(total)
+        if total && total.to_i > 0
+          @download_progress_bar = ProgressBar.create title: "Downloading elasticsearch #{version}", total: total,
+                                                      format: '%t |%bᗧ%i| %p%% (%r KB/sec) %e', progress_mark: ' ', remainder_mark: '･',
+                                                      rate_scale: ->(rate) { rate / 1024 }, smoothing: 0.7
+        end
+      end
+
+      def increment_progress(size)
+        @download_progress_bar.progress = size if @download_progress_bar
+      end
+
+    end
+
+  end
+end

data/lib/elasticsearch/embedded/logger_configuration.rb

@@ -0,0 +1,62 @@
+require 'logging'
+
+module Elasticsearch
+  module Embedded
+
+    # Contains stuff related to logging configuration
+    module LoggerConfiguration
+
+      # Configure logger verbosity for ::Elasticsearch::Embedded log hierarchy
+      # @param [String,Fixnum] level accepts strings levels or numbers
+      def verbosity(level)
+        level = level.to_s.downcase # normalize string to downcase
+        case
+          when level =~ /\A\d\Z/
+            Logging.logger[self].level = level.to_i
+          when Logging::LEVELS.include?(level)
+            Logging.logger[self].level = Logging::LEVELS[level]
+          else
+            Logging.logger[self].level = :info
+        end
+
+      end
+
+      # Clear all logging appenders for ::Elasticsearch::Embedded log hierarchy
+      def mute!
+        Logging.logger[self].clear_appenders
+      end
+
+      # @see https://github.com/TwP/logging/blob/master/examples/colorization.rb
+      def configure_logging!
+        # Configure logger levels for hierarchy
+        Logging.logger[self].level = :info
+        # Register a color scheme named bright
+        Logging.color_scheme 'bright', {
+            levels: {
+                info: :green,
+                warn: :yellow,
+                error: :red,
+                fatal: [:white, :on_red]
+            },
+            date: :blue,
+            logger: :cyan,
+            message: :white,
+        }
+        pattern_options = {pattern: '[%d] %-5l %c: %m\n'}
+        # Apply colors only if in tty
+        pattern_options[:color_scheme] = 'bright' if STDOUT.tty?
+        # Create a named appender to be used only for current module
+        Logging.logger[self].appenders = Logging.appenders.stdout(self.to_s, layout: Logging.layouts.pattern(pattern_options))
+      end
+
+      private
+
+      # Extension callback
+      def self.extended(base)
+        base.configure_logging!
+      end
+
+    end
+
+  end
+end

data/lib/elasticsearch/embedded/rspec_configuration.rb

@@ -0,0 +1,70 @@
+module Elasticsearch
+  module Embedded
+
+    module RSpec
+
+      module ElasticSearchHelpers
+        # Return a client connected to the configured client, if ::Elasticsearch::Client is defined
+        # return a client, else return a URI attached to cluster.
+        # @see http://ruby-doc.org/stdlib-2.1.2/libdoc/net/http/rdoc/Net/HTTP.html
+        # @see https://github.com/elasticsearch/elasticsearch-ruby
+        def client
+          @client ||=
+              case
+                when defined?(::Elasticsearch::Client)
+                  ::Elasticsearch::Client.new host: "localhost:#{cluster.port}"
+                else
+                  URI("http://localhost:#{cluster.port}/")
+              end
+        end
+
+        # Return a cluster instance to be used in tests
+        def cluster
+          ElasticSearchHelpers.memoized_cluster
+        end
+
+        class << self
+
+          # Return a singleton instance of cluster object
+          def memoized_cluster
+            @cluster ||= ::Elasticsearch::Embedded::Cluster.new
+          end
+
+        end
+
+      end
+
+      class << self
+
+        # Configure rspec for usage with ES cluster
+        def configure_with(*meta)
+          # assign default value to tags
+          ::RSpec.configure do |config|
+
+            # Include helpers only in tagged specs
+            config.include ElasticSearchHelpers, *meta
+
+            # Before hook, starts the cluster
+            config.before(:each, *meta) do
+              ElasticSearchHelpers.memoized_cluster.ensure_started!
+              ElasticSearchHelpers.memoized_cluster.delete_all_indices!
+            end
+
+            # After suite hook, stop the cluster
+            config.after(:suite) do
+              ElasticSearchHelpers.memoized_cluster.stop if ElasticSearchHelpers.memoized_cluster.running?
+            end
+          end
+        end
+
+        # Default config method, configure RSpec with :elasticsearch filter only.
+        # Equivalent to .configure_with(:elasticsearch)
+        def configure
+          configure_with(:elasticsearch)
+        end
+      end
+
+    end
+
+  end
+end

data/lib/elasticsearch/embedded/version.rb

@@ -0,0 +1,5 @@
+module Elasticsearch
+  module Embedded
+    VERSION = '0.1.0'
+  end
+end

data/spec/lib/elasticsearch/embedded/cluster_spec.rb

@@ -0,0 +1,93 @@
+describe Elasticsearch::Embedded::Cluster, :elasticsearch do
+
+  describe '#clear_all_indices!' do
+
+    it 'should delete all indices' do
+      # Create a document and an index
+      client.index index: 'test', type: 'test-type', id: 1, body: {title: 'Test'}
+      expect {
+        cluster.delete_all_indices!
+      }.to change { client.indices.get_settings.count }.to(0)
+    end
+
+    it 'should delete all indices' do
+      # Create a document and an index
+      client.index index: 'test', type: 'test-type', id: 1, body: {title: 'Test'}
+      client.index index: 'test2', type: 'test-type', id: 1, body: {title: 'Test'}
+      expect {
+        cluster.delete_index! 'test'
+      }.to change { client.indices.get_settings.keys }.to(['test2'])
+    end
+
+  end
+
+  describe '#pids' do
+
+    it 'should return a list running instances pids' do
+      expect(cluster.pids).to_not be_empty
+    end
+
+    it 'should return an empty array when not started' do
+      not_started = Elasticsearch::Embedded::Cluster.new
+      not_started.port = 50000 # Likely there's no process running on this port
+      expect(not_started.pids).to eq([])
+    end
+
+  end
+
+  describe 'Development template' do
+
+    before do
+      # It's not applied by default on non persistent clusters
+      cluster.apply_development_template!
+      client.indices.create index: 'any_index'
+    end
+
+    let(:index_settings) { client.indices.get_settings(index: 'any_index')['any_index']['settings']['index'] }
+
+    # Do not leave status across tests
+    after(:all) do
+      client.indices.delete_template name: 'development_template' if client.indices.get_template.has_key?('development_template')
+    end
+
+    it 'should configure 1 shard for each index' do
+      expect(index_settings).to include('number_of_shards' => '1')
+    end
+
+    it 'should configure 0 replicas for each index' do
+      expect(index_settings).to include('number_of_replicas' => '0')
+    end
+
+  end
+
+  describe 'Cluster persistency' do
+
+    let(:persistent_cluster) { Elasticsearch::Embedded::Cluster.new }
+    let(:port) { cluster.port + 10 }
+    let(:client) { Elasticsearch::Client.new host: "localhost:#{port}" }
+
+    before do
+      persistent_cluster.persistent = true
+      persistent_cluster.cluster_name = 'persistent_cluster'
+      persistent_cluster.port = port
+    end
+
+    after do
+      # Ensure additional cluster is stopped when test is finished
+      persistent_cluster.stop if persistent_cluster.running?
+    end
+
+    it 'should persist data across restarts' do
+      persistent_cluster.start
+      # Index a document and trigger persistent index creation
+      client.index index: 'persistent', type: 'test-type', id: 1, body: {title: 'Test'}, refresh: true
+      # Restart cluster and check index presence
+      expect {
+        persistent_cluster.stop
+        persistent_cluster.start
+      }.to_not change { client.indices.get_settings }
+    end
+
+  end
+
+end

data/spec/lib/elasticsearch/embedded/downloader_spec.rb

@@ -0,0 +1,131 @@
+describe Elasticsearch::Embedded::Downloader do
+
+  describe 'configuration using env' do
+
+    it 'should allow configuration through ENV' do
+      ENV['ELASTICSEARCH_VERSION'] = '1.3.3'
+      ENV['ELASTICSEARCH_DOWNLOAD_PATH'] = '/tmp'
+      d = Elasticsearch::Embedded::Downloader.new
+      expect(d.version).to eq '1.3.3'
+      expect(d.path).to eq '/tmp'
+    end
+
+    after(:each) do
+      ENV['ELASTICSEARCH_VERSION'] = nil
+      ENV['ELASTICSEARCH_DOWNLOAD_PATH'] = nil
+    end
+
+  end
+
+  describe 'configuration using options' do
+
+    it 'should allow configuration using option arguments' do
+      d = Elasticsearch::Embedded::Downloader.new(path: '/tmp', version: '1.3.3')
+      expect(d.version).to eq '1.3.3'
+      expect(d.path).to eq '/tmp'
+    end
+
+  end
+
+  describe 'default configuration' do
+
+    it 'should use default values' do
+      d = Elasticsearch::Embedded::Downloader.new
+      expect(d.version).to eq Elasticsearch::Embedded::Downloader::DEFAULT_VERSION
+      expect(d.path).to eq Elasticsearch::Embedded::Downloader::TEMPORARY_PATH
+    end
+
+  end
+
+  # Integration tests
+  context 'with mocked filesystem' do
+
+    # Stub filesystem calls
+    include FakeFS::SpecHelpers
+
+    let(:fake_archive) do
+      directory = "/elasticsearch-#{subject.version}"
+      archive = "/elasticsearch-#{subject.version}.zip"
+      FileUtils.mkdir_p(directory)
+      FileUtils.touch(File.join(directory, 'README.textile'))
+      FileUtils.mkdir_p(File.join(directory, 'bin'))
+      FileUtils.touch(File.join(directory, 'bin', 'elasticsearch'))
+      # Creates a zip archive with empty files
+      Zip::File.open(archive, Zip::File::CREATE) do |zipfile|
+        # make root folder of zip archive
+        zipfile.add(directory.sub('/', ''), directory)
+        # Add all content
+        Dir[File.join(directory, '**', '**')].each do |file|
+          zipfile.add(file.sub('/', ''), file)
+        end
+      end
+      # Return the built archive file name
+      archive
+    end
+
+    # Realpath is needed because mockfs does not implement realpath
+    let(:path) { File.realpath(Dir.tmpdir) }
+    let(:zip_file) { File.join(path, "elasticsearch-#{subject.version}.zip") }
+    let(:dist_folder) { File.join(path, "elasticsearch-#{subject.version}") }
+    let(:executable) { File.join(dist_folder, 'bin', 'elasticsearch') }
+    subject { Elasticsearch::Embedded::Downloader.new(path: path) }
+
+    before do
+      # This is needed on OS X where /tmp is a symlink otherwise the
+      # openuri call will fail with Errno::ENOENT when trying to create a temporary file
+      FileUtils.mkdir_p(Dir.tmpdir)
+      # Ensure download path is present on the stubbed filesystem
+      FileUtils.mkdir_p(path)
+    end
+
+    describe 'file download' do
+
+      it 'should download file into configured path' do
+        expect(File.exists?(zip_file)).to be_falsey
+        # Stub only actual download of file
+        expect(OpenURI).to receive(:open_uri).and_return(File.open(fake_archive))
+        subject.perform
+        expect(File.size(zip_file)).to be >= 0
+      end
+
+      it 'should not download file if already present' do
+        FileUtils.touch(zip_file) # Simulate presence of downloaded version
+        FileUtils.mkdir_p(dist_folder) # Simulate presence of extracted version
+        expect { subject.perform }.to_not change { File.size(zip_file) }
+      end
+
+    end
+
+    describe 'file extraction' do
+
+      before do
+        # create the zip archive as if it were downloaded
+        expect(File.exists?(fake_archive)).to be_truthy
+        FileUtils.cp fake_archive, zip_file
+      end
+
+      it 'should extract zip archive when into final folder' do
+        expect { subject.perform }.to change { Dir[File.join(dist_folder, '**', '**')].count }
+      end
+
+      it 'should not extract zip archive when final folder is already present' do
+        FileUtils.mkdir_p dist_folder
+        expect { subject.perform }.to_not change { Dir[File.join(dist_folder, '**', '**')] }
+      end
+
+      it 'should make bin/elasticsearch executable' do
+        subject.perform
+        expect(File.executable?(executable)).to be_truthy
+      end
+
+      it 'should create a folder for log files' do
+        expect {
+          subject.perform
+        }.to change { File.directory?(File.join(dist_folder, 'logs')) }
+      end
+
+    end
+
+  end
+
+end