em-stretcher 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: baa41f4d1deadbbb5bac9415dc9f6800fa89cff7
+  data.tar.gz: 79dd2a858e7cbcc507d45f50b20632656ed0635f
+SHA512:
+  metadata.gz: e6e01ba6d3c896b6ccaadf3c519983bac59946c8ee3e3b572ec0557b9c6f9d40d9d5475737e6dd59f14aa6bca8720883f4476604b6967cd961602a957877e94b
+  data.tar.gz: 4b626de495880b06323803c8e6404e1244765415b0268d18459a3c6d0f301ed2ca1d58d7d765975ce0e5a1c137e1a5a6b435fff15a442a501ed292835f49a9dc

data/.gitignore

@@ -0,0 +1,18 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp
+test.rb

data/.rspec

@@ -0,0 +1,3 @@
+--pattern "**/spec/**/*_spec.rb" --default-path .
+--color
+--format documentation

data/.ruby-version

@@ -0,0 +1 @@
+ruby-2.0.0-p353

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in em-stretcher.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2014 Benjamin Coe
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,151 @@
+Em::Stretcher
+=============
+
+An EventMachine port of [Stretcher](https://github.com/PoseBiz/stretcher) (a Fast, Elegant, ElasticSearch client.)
+
+Indexes
+-------
+
+**Creating an Index**
+
+```ruby
+require 'eventmachine'
+require "em-stretcher"
+
+EM.run do
+  server = EM::Stretcher::Server.new
+
+  server.index('my-index').create
+    .callback do |result|
+      p result
+    end
+    .errback do |err|
+      p err
+    end
+end
+```
+
+**Deleting an Index**
+
+```ruby
+server.index('my-index').delete
+	.callback do |result|
+	  p result
+	end
+	.errback do |err|
+	  p err
+	end
+```
+
+Mappings
+--------
+
+**Creating a Mapping**
+
+```ruby
+mapping = {
+  :article => {
+    properties: {
+      title: { :type => :string }
+    }
+  }
+}
+
+server.index('my-index')
+	.type('article')
+	.put_mapping(mapping)
+	.callback do |result|
+		p result
+	end
+	.errback do |err|
+		p err
+	end
+```
+
+Indexing Documents
+------------------
+
+```ruby
+id = 33
+
+server.index('my-index')
+  .type(mapping_name)
+  .put(id, { title: "My Document with the id #{id}" })
+  .callback do |response|
+    p response
+  end
+  .errback do |err|
+  	p err
+  end
+```
+
+Searching for Documents
+-----------------------
+
+```ruby
+server.index('my-index').type('articles')
+	.search(size: 50, query: { "query_string" => { "query" => "*" } })
+	.documents
+	.callback do |r|
+		p r
+	end
+```
+
+When One Thing Leads to Another
+-------------------------------
+
+EM::Stretcher uses [Deferrable Gratification](https://github.com/samstokes/deferrable_gratification) for chaining together dependent requests.
+
+Here's a great example of when this comes into play:
+
+* You index several hundred documents in parallel.
+* You want to perform a search on the index, with all the documents present.
+
+Here's how you can pull this off with Deferrable Gratification:
+
+```ruby
+# Index some documents in parallel.
+(0..50).each do |i|
+	server.index('my-index')
+	  .type('articles')
+	  .put(i, { title: "title #{i}" })
+	  .callback do |response|
+	    p response
+	  end
+end
+
+# First we wait for the indexing to finish.
+server.index('my-index').refresh
+	.bind! do
+
+	  # Then we perform the search.
+	  server.index('my-index').type('articles')
+	    .search(size: 50, query: { "query_string" => { "query" => "*" } })
+	    .documents
+	    .callback do |r|
+	      p r # all 50 results should have been returned.
+	    end
+	end
+```
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+    gem 'em-stretcher'
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install em-stretcher
+
+## Contributing
+
+1. Fork it ( http://github.com/bcoe/em-stretcher/fork )
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new Pull Request

data/Rakefile

@@ -0,0 +1,4 @@
+require "bundler/gem_tasks"
+require "rspec/core/rake_task"
+
+RSpec::Core::RakeTask.new(:spec)

data/em-stretcher.gemspec

@@ -0,0 +1,29 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'em/stretcher/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "em-stretcher"
+  spec.version       = EventMachine::Stretcher::VERSION
+  spec.authors       = ["Benjamin Coe"]
+  spec.email         = ["ben@yesware.com"]
+  spec.summary       = %q{EventMachine for Stretcher a Fast, Elegant, ElasticSearch client}
+  spec.homepage      = ""
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files`.split($/)
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+
+  spec.add_dependency "stretcher"
+  spec.add_dependency "deferrable_gratification"
+  spec.add_dependency "eventmachine"
+  spec.add_dependency "em-http-request"
+
+  spec.add_development_dependency "bundler", "~> 1.5"
+  spec.add_development_dependency "rspec"
+  spec.add_development_dependency "rspec-eventmachine"
+  spec.add_development_dependency "rake"
+end

data/lib/em-stretcher.rb

@@ -0,0 +1,19 @@
+require "ostruct"
+require "stretcher"
+require "eventmachine"
+require "deferrable_gratification"
+require "em-http-request"
+require "em/stretcher/server"
+require "em/stretcher/version"
+require "em/stretcher/search_results"
+require "em/stretcher/index_type"
+
+module EventMachine
+  module Stretcher
+    # Your code goes here...
+  end
+end
+
+# Enchance deferrables with bind functionality.
+DG.enhance_all_deferrables!
+DG.enhance! EventMachine::HttpClient

data/lib/em/stretcher/index_type.rb

@@ -0,0 +1,22 @@
+# Patch get to play nicely with EM.
+module Stretcher
+  # Represents an index  scoped to a specific type.
+  # Generally should be instantiated via Index#type(name).
+  class IndexType
+    # Retrieves the document by ID.
+    # Normally this returns the contents of _source, however, if the 'raw' flag is passed in, it will return the full response hash.
+    # Returns nil if the document does not exist.
+    # 
+    # The :fields argument can either be a csv String or an Array. e.g. [:field1,'field2] or "field1,field2".
+    # If the fields parameter is passed in those fields are returned instead of _source.
+    #
+    # If, you include _source as a field, along with other fields you MUST set the raw flag to true to 
+    # receive both fields and _source. Otherwise, only _source will be returned
+    def get(id, options={})      
+      request(:get, id, options)
+        .bind! do |res|
+          res._source || res.fields
+        end
+    end
+  end
+end

data/lib/em/stretcher/search_results.rb

@@ -0,0 +1,32 @@
+# Patch search results to play nice with deferrables.
+module Stretcher
+  # Conveniently represents elastic search results in a more compact fashion
+  #
+  # Available properties:
+  #
+  # * raw : The raw response from elastic search
+  # * total : The total number of matched docs
+  # * facets : the facets hash
+  # * results : The hit results with _id merged in to _source
+  class SearchResults
+    # Returns a 'prettier' version of elasticsearch results
+    # Also aliased as +docs+
+    # This will:
+    #
+    # 1. Return either '_source' or 'fields' as the base of the result
+    # 2. Merge any keys beginning with a '_' into it as well (such as '_score')
+    # 3. Copy the 'highlight' field into '_highlight'
+    #
+    def documents
+      raw_plain.bind! do |result|
+        result.hits.hits.map do |hit|
+          doc = extract_source(hit)
+          copy_underscores(hit, doc)
+          copy_highlight(hit, doc)
+          doc
+        end
+      end
+    end
+    alias_method :docs, :documents
+  end
+end

data/lib/em/stretcher/server.rb

@@ -0,0 +1,85 @@
+# Patch the stretcher server to use
+# deferrables rather than Faraday.
+module EventMachine::Stretcher
+  class Server < Stretcher::Server
+    # Handy way to query the server, returning *only* the body
+    # Will fail with an exception when the status is not in the 2xx range.
+    #
+    # @param method [Symbol] HTTP method to execute.
+    # @param path [String] full ES API URL to hit (http://127.0.0.1:9200/_status).
+    # @param body [Hash] Hash ElasticSearch query body.
+    # @param headers [Hash] additional headers.
+    # @param options [Options]
+    #  @param options.mashify [Boolean] should the response be turned into a Hashie::Mash?
+    # @return [Deferrable] deferrable that yields ElasticSearch response.
+    def request(method, path, params={}, body=nil, headers={}, options={})
+      options = { :mashify => true }.merge(options)
+
+      # Rather than setting up the default headers using the
+      # Faraday middlewear, we set them here.
+      http_params = {
+        headers: headers.merge({
+          :accept =>  'application/json',
+          :user_agent => "Stretcher Ruby Gem #{Stretcher::VERSION}",
+          "Content-Type" => "application/json"
+        })
+      }
+
+      deferrable = EventMachine::DefaultDeferrable.new
+
+      # Allow the body to be set via a block
+      # es_component.rb in the stretcher library.
+      body_struct = OpenStruct.new
+      yield(body_struct) if block_given?
+      body ||= body_struct.body
+      
+      # Update params with GET and POST parameters.
+      http_params[:query] = Stretcher::Util.clean_params(params) if params
+      http_params[:body] = JSON.dump(body) if body
+
+      # Execute :get, :post, :put, or :delete, returns a deferrable.
+      http = EventMachine::HttpRequest.new(path).send(method,  http_params)
+
+      # We return our own deferrable, so that we can parse results.
+      http.callback do
+        check_response(http, deferrable, options)
+      end
+
+      http.errback do |err|
+        deferrable.fail(err)
+      end
+
+      deferrable
+    end
+
+    private
+
+    # Internal use only
+    # Check response codes from request
+    def check_response(res, deferrable, options)
+      status = res.response_header.status
+
+      if status >= 200 && status <= 299
+        if(options[:mashify])
+          begin
+            parsed_response = JSON.parse(res.response)
+            deferrable.succeed(Hashie::Mash.new(parsed_response))
+          rescue
+            deferrable.succeed(res.response)
+          end
+        else
+          deferrable.succeed(res.response)
+        end
+      elsif [404, 410].include? status
+        err_str = "Error processing request: (#{status})! #{res.req.method} URL: #{res.req.uri}"
+        err_str << "\n Resp Body: #{res.response}"
+        deferrable.fail(Stretcher::RequestError::NotFound.new(err_str))
+      else
+        err_str = "Error processing request (#{status})! #{res.req.method} URL: #{res.req.uri}"
+        err_str << "\n Resp Body: #{res.response}"
+        deferrable.fail(Stretcher::RequestError.new(err_str))
+      end
+    end
+
+  end
+end

data/lib/em/stretcher/version.rb

@@ -0,0 +1,5 @@
+module EventMachine
+  module Stretcher
+    VERSION = "0.0.1"
+  end
+end

data/spec/server_spec.rb

@@ -0,0 +1,111 @@
+require "spec_helper"
+
+describe EventMachine::Stretcher::Server do
+    
+  include DeferrableModule
+
+  before(:all) { reset_index }
+  before(:each) { execute(type, :put_mapping, mapping) }
+  
+  let(:server) { EventMachine::Stretcher::Server.new }
+  let(:index) { server.index(TESTING_INDEX_NAME) }
+  let(:type) { index.type(mapping_name) }
+  let(:mapping_name) { :article }
+  let(:mapping) do
+    {
+      mapping_name => {
+        properties: {
+          title: { :type => :string }
+        }
+      }
+    }
+  end
+  let(:document) do 
+    {title: 'hello world!' }
+  end
+
+  describe "index" do
+    it "can delete an index" do
+      execute(index, :delete)
+      execute(index, :status)
+      error_response_should_contain 'IndexMissingException'
+    end
+
+    it "can create an index" do
+      execute(index, :delete)
+      execute(index, :create, {
+        :settings => {
+          :number_of_shards => 3,
+          :number_of_replicas => 0 
+        }
+      })
+      execute(index, :status)
+      success_key_should_have_value([:_shards, :total], 3)
+    end
+  end
+
+  context "creating mappings" do
+    it "can create a mapping" do
+      type = index.type(mapping_name)
+      execute(type, :put_mapping, mapping)
+      execute(index, :get_mapping)
+
+      success_key_should_have_value([
+        TESTING_INDEX_NAME.to_sym,
+        mapping_name
+      ], { "properties" => { "title" => { "type"=>"string" } } })
+    end
+  end
+
+  context "documents" do
+    it "can index a document" do
+      execute(type, :put, 1, document)
+      execute(index, :refresh)
+      execute(type, :search)
+      success_result_count_should_equal(1)
+    end
+
+    it "can fetch a document by its id" do
+      execute(type, :put, 1, document)
+      execute(index, :refresh)
+      execute(type, :get, 1)
+
+      success_key_should_have_value(:title, 'hello world!')
+    end
+
+    it "should return a 404 if document is not found" do
+      execute(type, :get, 2)
+
+      error_response_should_contain '404'
+    end
+
+    it "can delete a document" do
+      execute(type, :put, 1, document)
+      execute(index, :refresh)
+      execute(type, :delete, 1, document)
+      execute(index, :refresh)
+
+      execute(type, :search)
+      success_result_count_should_equal(0)
+    end
+  end
+
+  describe "search" do
+    it "returns a document if query matches title" do
+      execute(type, :put, 1, document)
+      execute(index, :refresh)
+      execute(type, :search, query: { "query_string" => { "query" => "hello world" } })
+
+      success_result_count_should_equal(1)
+    end
+
+    it "does not return a document if query does not match title" do
+      execute(type, :put, 1, document)
+      execute(index, :refresh)
+      execute(type, :search, query: { "query_string" => { "query" => "zebra" } })
+
+      success_result_count_should_equal(0)
+    end
+  end
+
+end

data/spec/spec_helper.rb

@@ -0,0 +1,97 @@
+require "rspec"
+require "rspec/em"
+
+require_relative '../lib/em-stretcher'
+
+TESTING_INDEX_NAME = 'em::stretcher:testing'
+
+# Helper to reset ElasticSearch index for tests.
+def reset_index
+  server = Stretcher::Server.new
+  i = server.index(TESTING_INDEX_NAME)
+  begin
+    i.delete
+  rescue Stretcher::RequestError::NotFound
+  end
+  server.refresh
+  i.create({
+    :settings => {
+      :number_of_shards => 1,
+      :number_of_replicas => 0
+    }
+  })
+  # Why do both? Doesn't hurt, and it fixes some races
+  server.refresh
+  i.refresh
+  
+  attempts_left = 40
+  
+  # Sometimes the index isn't instantly available
+  loop do
+    idx_metadata = server.cluster.request(:get, :state)[:metadata][:indices][i.name]
+    i_state =  idx_metadata[:state]
+    
+    break if i_state == 'open'
+    
+    if attempts_left < 1
+        raise "Bad index state! #{i_state}. Metadata: #{idx_metadata}" 
+    end
+
+    sleep 0.1
+    attempts_left -= 1
+  end
+end
+
+# async assertions for EventMachine specs.
+DeferrableModule = RSpec::EM.async_steps do
+  def execute(object, method, *args, &callback)
+    deferrable = if args
+      object.send(method, *args)
+    else
+      object.send(method)
+    end
+
+    # Search results have their deferrable
+    # on the documents key.
+    if deferrable.respond_to?(:documents)
+      deferrable = deferrable.results
+    end
+
+    deferrable.callback do |result|
+      @success = result
+      callback.call
+    end
+
+    deferrable.errback do |err|
+      @error = err
+      callback.call
+    end
+  end
+
+  # recursively look for a matching key and value in hash.
+  def success_key_should_have_value(key, expected, &callback)
+    # walk the hash structure.
+    key = [*key]
+    while key.count > 0
+      @success = @success.send(key.shift)
+    end
+
+    @success.should == expected
+    @success = nil
+    callback.call
+  end
+
+  # confirm count of result set.
+  def success_result_count_should_equal(expected, &callback)
+    @success.count.should == expected
+    @success = nil
+    callback.call
+  end
+
+  # look for substring in error http_response.
+  def error_response_should_contain(expected, &callback)
+    @error.http_response.should =~ /#{expected}/
+    @error = nil
+    callback.call
+  end
+end

metadata

@@ -0,0 +1,174 @@
+--- !ruby/object:Gem::Specification
+name: em-stretcher
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- Benjamin Coe
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2014-03-11 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: stretcher
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: deferrable_gratification
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: eventmachine
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: em-http-request
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.5'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.5'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rspec-eventmachine
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: 
+email:
+- ben@yesware.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- .rspec
+- .ruby-version
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- em-stretcher.gemspec
+- lib/em-stretcher.rb
+- lib/em/stretcher/index_type.rb
+- lib/em/stretcher/search_results.rb
+- lib/em/stretcher/server.rb
+- lib/em/stretcher/version.rb
+- spec/server_spec.rb
+- spec/spec_helper.rb
+homepage: ''
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.2.1
+signing_key: 
+specification_version: 4
+summary: EventMachine for Stretcher a Fast, Elegant, ElasticSearch client
+test_files:
+- spec/server_spec.rb
+- spec/spec_helper.rb
+has_rdoc: