elastomer-client 0.3.1

data/lib/elastomer/middleware/opaque_id.rb

@@ -0,0 +1,69 @@
+require 'securerandom'
+
+module Elastomer
+  module Middleware
+
+    # This Faraday middleware implements the "X-Opaque-Id" request / response
+    # headers for ElasticSearch. The X-Opaque-Id header, when provided on the
+    # request header, will be returned as a header in the response. This is
+    # useful in environments which reuse connections to ensure that cross-talk
+    # does not occur between two requests.
+    #
+    # The SecureRandom lib is used to generate a UUID string for each request.
+    # This value is used as the content for the "X-Opaque-Id" header. If the
+    # value is different between the request and the response, then an
+    # `Elastomer::Client::OpaqueIdError` is raised. In this case no response
+    # will be returned.
+    #
+    # See [ElasticSearch "X-Opaque-Id"
+    # header](https://github.com/elasticsearch/elasticsearch/issues/1202)
+    # for more details.
+    class OpaqueId < ::Faraday::Middleware
+      X_OPAQUE_ID = 'X-Opaque-Id'.freeze
+      COUNTER_MAX = 2**32 - 1
+
+      # Faraday middleware implementation.
+      #
+      # env - Faraday environment Hash
+      #
+      # Returns the environment Hash
+      def call( env )
+        uuid = generate_uuid.freeze
+        env[:request_headers][X_OPAQUE_ID] = uuid
+
+        @app.call(env).on_complete do |renv|
+          if uuid != renv[:response_headers][X_OPAQUE_ID]
+            raise ::Elastomer::Client::OpaqueIdError, "conflicting 'X-Opaque-Id' headers"
+          end
+        end
+      end
+
+      # Generate a UUID using the built-in SecureRandom class. This can be a
+      # little slow at times, so we will reuse the same UUID and append an
+      # incrementing counter.
+      #
+      # Returns the UUID string.
+      def generate_uuid
+        t = Thread.current
+
+        unless t.key? :opaque_id_base
+          t[:opaque_id_base]    = (SecureRandom.urlsafe_base64(12) + '%08x').freeze
+          t[:opaque_id_counter] = -1
+        end
+
+        t[:opaque_id_counter] += 1
+        t[:opaque_id_counter] = 0 if t[:opaque_id_counter] > COUNTER_MAX
+        t[:opaque_id_base] % t[:opaque_id_counter]
+      end
+
+    end  # OpaqueId
+  end  # Middleware
+
+  # Error raised when a conflict is detected between the UUID sent in the
+  # 'X-Opaque-Id' request header and the one received in the response header.
+  Client::OpaqueIdError = Class.new Client::Error
+
+end  # Elastomer
+
+Faraday::Request.register_middleware \
+  :opaque_id => ::Elastomer::Middleware::OpaqueId

data/lib/elastomer/middleware/parse_json.rb

@@ -0,0 +1,39 @@
+module Elastomer
+  module Middleware
+
+    # Parse response bodies as JSON.
+    class ParseJson < Faraday::Middleware
+      CONTENT_TYPE = 'Content-Type'.freeze
+      MIME_TYPE    = 'application/json'.freeze
+
+      def call(environment)
+        @app.call(environment).on_complete do |env|
+          if process_response?(env)
+            env[:body] = parse env[:body]
+          end
+        end
+      end
+
+      # Parse the response body.
+      def parse(body)
+        MultiJson.load(body) if body.respond_to?(:to_str) and !body.strip.empty?
+      rescue StandardError, SyntaxError => e
+        raise Faraday::Error::ParsingError, e
+      end
+
+      def process_response?(env)
+        type = response_type(env)
+        type.empty? or type == MIME_TYPE
+      end
+
+      def response_type(env)
+        type = env[:response_headers][CONTENT_TYPE].to_s
+        type = type.split(';', 2).first if type.index(';')
+        type
+      end
+    end
+  end
+end
+
+Faraday::Response.register_middleware \
+  :parse_json => ::Elastomer::Middleware::ParseJson

data/lib/elastomer/notifications.rb

@@ -0,0 +1,83 @@
+require 'active_support/notifications'
+require 'securerandom'
+require 'elastomer/client'
+
+module Elastomer
+
+  # So you want to get notifications from your ElasticSearch client? Well,
+  # you've come to the right place!
+  #
+  #   require 'elastomer/notifications'
+  #
+  # Requiring this module will add ActiveSupport notifications to all
+  # ElasticSearch requests. To subscribe to those requests ...
+  #
+  #   ActiveSupport::Notifications.subscribe('request.client.elastomer') do |name, start_time, end_time, _, payload|
+  #     duration = end_time - start_time
+  #     $stderr.puts '[%s] %s %s (%.3f)' % [payload[:status], payload[:index], payload[:action], duration]
+  #   end
+  #
+  # The payload contains the following bits of information:
+  #
+  # * :index  - index name (if any)
+  # * :type   - documeht type (if any)
+  # * :action - the action being performed
+  # * :url    - request URL
+  # * :method - request method (:head, :get, :put, :post, :delete)
+  # * :status - response status code
+  #
+  # If you want to use your own notifications service then you will need to
+  # let Elastomer know by setting the `service` here in the Notifications
+  # module. The service should adhere to the ActiveSupport::Notifications
+  # specification.
+  #
+  #   Elastomer::Notifications.service = your_own_service
+  #
+  module Notifications
+
+    class << self
+      attr_accessor :service
+    end
+
+    # The name to subscribe to for notifications
+    NAME = 'request.client.elastomer'.freeze
+
+    # Internal: Execute the given block and provide instrumentaiton info to
+    # subscribers. The name we use for subscriptions is
+    # `request.client.elastomer` and a supplemental payload is provided with
+    # more information about the specific ElasticSearch request.
+    #
+    # path   - The full request path as a String
+    # body   - The request body as a String or `nil`
+    # params - The request params Hash
+    # block  - The block that will be instrumented
+    #
+    # Returns the response from the block
+    def instrument( path, body, params )
+      payload = {
+        :index   => params[:index],
+        :type    => params[:type],
+        :action  => params[:action],
+        :context => params[:context],
+        :body    => body
+      }
+
+      ::Elastomer::Notifications.service.instrument(NAME, payload) do
+        response = yield
+        payload[:url]    = response.env[:url]
+        payload[:method] = response.env[:method]
+        payload[:status] = response.status
+        response
+      end
+    end
+  end
+
+  # use ActiveSupport::Notifications as the default instrumentaiton service
+  Notifications.service = ActiveSupport::Notifications
+
+  # inject our instrument method into the Client class
+  class Client
+    remove_method :instrument
+    include ::Elastomer::Notifications
+  end
+end

data/lib/elastomer/version.rb

@@ -0,0 +1,7 @@
+module Elastomer
+  VERSION = '0.3.1'
+
+  def self.version
+    VERSION
+  end
+end

data/script/bootstrap

@@ -0,0 +1,16 @@
+#!/bin/sh
+set -e
+
+# force use of gcc under mac due to hiredis issues under clang
+if [ $(uname) = Darwin ]; then
+    export CC=/usr/bin/gcc
+    export CXX=/usr/bin/g++
+    export LD=/usr/bin/gcc
+fi
+
+cd "$(dirname "$0")/.."
+if bundle check 1>/dev/null 2>&1; then
+    echo "Gem environment up-to-date"
+else
+    exec bundle install --binstubs --path vendor/gems "$@"
+fi

data/script/cibuild

@@ -0,0 +1,28 @@
+#!/bin/sh
+# Usage: script/cibuild
+# CI build script.
+# This is tailored for the janky build machines.
+set -e
+
+# change into root dir and setup path
+cd $(dirname "$0")/..
+PATH="$(pwd)/bin:$(pwd)/script:/usr/share/rbenv/shims:$PATH"
+
+# Write commit we're building at
+git log -n 1 || true
+echo
+
+result=0
+
+export RBENV_VERSION="$(cat .ruby-version)"
+echo "Building under $RBENV_VERSION"
+script/bootstrap
+script/testsuite || result=$?
+
+if [ $result -ne 0 ]; then
+    exit $result
+fi
+
+# echo
+# echo "Running benchmarks"
+# script/benchmark

data/script/console

@@ -0,0 +1,9 @@
+#!/usr/bin/env ruby
+require 'irb'
+require 'rubygems'
+require 'bundler/setup'
+
+$LOAD_PATH.unshift 'lib'
+require 'elastomer/client'
+
+IRB.start

data/script/testsuite

@@ -0,0 +1,10 @@
+#!/bin/sh
+set -e
+cd "$(dirname "$0")/.."
+
+# run entire test suite
+ruby --version 1>&2
+exec ruby -I. -rubygems \
+    -r "$(pwd)/test/test_helper" \
+    -e "ARGV.each { |f| require(f) }" \
+    -- $(find test -name '*_test.rb')

data/test/assertions.rb

@@ -0,0 +1,74 @@
+module Minitest::Assertions
+  #COMPATIBILITY
+  # ES 1.0 replaced the 'ok' attribute with a 'created' attribute
+  # in index responses. Check for either one so we are compatible
+  # with 0.90 and 1.0.
+  def assert_created(response)
+    assert response['created'] || response['ok'], 'document was not created'
+  end
+
+  #COMPATIBILITY
+  # ES 1.0 replaced the 'ok' attribute with an 'acknowledged' attribute
+  # in many responses. Check for either one so we are compatible
+  # with 0.90 and 1.0.
+  def assert_acknowledged(response)
+    assert response['acknowledged'] || response['ok'], 'document was not acknowledged'
+  end
+
+  #COMPATIBILITY
+  # ES 1.0 replaced the 'exists' attribute with a 'found' attribute in the
+  # get document response. Check for either one so we are compatible
+  # with 0.90 and 1.0.
+  def assert_found(response)
+    assert response['found'] || response['exists'], 'document was not found'
+  end
+
+  def refute_found(response)
+    refute response['found'] || response['exists'], 'document was unexpectedly found'
+  end
+
+  #COMPATIBILITY
+  # ES 1.0 replaced the 'ok' attribute in the bulk response item with a
+  # 'status' attribute. Here we check for either one for compatibility
+  # with 0.90 and 1.0.
+  def assert_bulk_index(item, message='bulk index did not succeed')
+    ok     = item['index']['ok']
+    status = item['index']['status']
+    assert ok == true || status == 201, message
+  end
+
+  def assert_bulk_create(item, message='bulk create did not succeed')
+    ok     = item['create']['ok']
+    status = item['create']['status']
+    assert ok == true || status == 201, message
+  end
+
+  def assert_bulk_delete(item, message='bulk delete did not succeed')
+    ok     = item['delete']['ok']
+    status = item['delete']['status']
+    assert ok == true || status == 200, message
+  end
+
+  #COMPATIBILITY
+  # ES 1.0 nests mappings in a "mappings" element under the index name, e.g.
+  # mapping["test-index"]["mappings"]["doco"]
+  # ES 0.90 doesn't have the "mappings" element:
+  # mapping["test-index"]["doco"]
+  def assert_mapping_exists(response, type, message="mapping expected to exist, but doesn't")
+    mapping = if response.has_key?('mappings')
+      response['mappings'][type]
+    else
+      response[type]
+    end
+    refute_nil mapping, message
+  end
+
+  def assert_property_exists(response, type, property, message="property expected to exist, but doesn't")
+    mapping = if response.has_key?('mappings')
+      response['mappings'][type]
+    else
+      response[type]
+    end
+    assert mapping['properties'].has_key?(property), message
+  end
+end

data/test/client/bulk_test.rb

@@ -0,0 +1,226 @@
+require File.expand_path('../../test_helper', __FILE__)
+
+describe Elastomer::Client::Bulk do
+
+  before do
+    @name  = 'elastomer-bulk-test'
+    @index = $client.index(@name)
+
+    unless @index.exists?
+      @index.create \
+        :settings => { 'index.number_of_shards' => 1, 'index.number_of_replicas' => 0 },
+        :mappings => {
+          :tweet => {
+            :_source => { :enabled => true }, :_all => { :enabled => false },
+            :properties => {
+              :message => { :type => 'string', :analyzer => 'standard' },
+              :author  => { :type => 'string', :index => 'not_analyzed' }
+            }
+          },
+          :book => {
+            :_source => { :enabled => true }, :_all => { :enabled => false },
+            :properties => {
+              :title  => { :type => 'string', :analyzer => 'standard' },
+              :author => { :type => 'string', :index => 'not_analyzed' }
+            }
+          }
+        }
+
+      wait_for_index(@name)
+    end
+  end
+
+  after do
+    @index.delete if @index.exists?
+  end
+
+  it 'performs bulk index actions' do
+    body = [
+      '{"index" : {"_id":"1", "_type":"tweet", "_index":"elastomer-bulk-test"}}',
+      '{"author":"pea53", "message":"just a test tweet"}',
+      '{"index" : {"_id":"1", "_type":"book", "_index":"elastomer-bulk-test"}}',
+      '{"author":"John Scalzi", "title":"Old Mans War"}',
+      nil
+    ]
+    body = body.join "\n"
+    h = $client.bulk body
+
+    assert_bulk_index(h['items'][0])
+    assert_bulk_index(h['items'][1])
+
+    @index.refresh
+
+    h = @index.docs('tweet').get :id => 1
+    assert_equal 'pea53', h['_source']['author']
+
+    h = @index.docs('book').get :id => 1
+    assert_equal 'John Scalzi', h['_source']['author']
+
+
+    body = [
+      '{"index" : {"_id":"2", "_type":"book"}}',
+      '{"author":"Tolkien", "title":"The Silmarillion"}',
+      '{"delete" : {"_id":"1", "_type":"book"}}',
+      nil
+    ]
+    body = body.join "\n"
+    h = $client.bulk body, :index => @name
+
+    assert_bulk_index h['items'].first, 'expected to index a book'
+    assert_bulk_delete h['items'].last, 'expected to delete a book'
+
+    @index.refresh
+
+    h = @index.docs('book').get :id => 1
+    assert !h['exists'], 'was not successfully deleted'
+
+    h = @index.docs('book').get :id => 2
+    assert_equal 'Tolkien', h['_source']['author']
+  end
+
+  it 'supports a nice block syntax' do
+    h = @index.bulk do |b|
+      b.index :_id => 1,   :_type => 'tweet', :author => 'pea53', :message => 'just a test tweet'
+      b.index :_id => nil, :_type => 'book', :author => 'John Scalzi', :title => 'Old Mans War'
+    end
+    items = h['items']
+
+    assert_instance_of Fixnum, h['took']
+
+    assert_bulk_index h['items'].first
+    assert_bulk_create h['items'].last
+
+    book_id = items.last['create']['_id']
+    assert_match %r/^\S{22}$/, book_id
+
+    @index.refresh
+
+    h = @index.docs('tweet').get :id => 1
+    assert_equal 'pea53', h['_source']['author']
+
+    h = @index.docs('book').get :id => book_id
+    assert_equal 'John Scalzi', h['_source']['author']
+
+
+    h = @index.bulk do |b|
+      b.index  :_id => '', :_type => 'book', :author => 'Tolkien', :title => 'The Silmarillion'
+      b.delete :_id => book_id, :_type => 'book'
+    end
+    items = h['items']
+
+    assert_bulk_create h['items'].first, 'expected to create a book'
+    assert_bulk_delete h['items'].last, 'expected to delete a book'
+
+    book_id2 = items.first['create']['_id']
+    assert_match %r/^\S{22}$/, book_id2
+
+    @index.refresh
+
+    h = @index.docs('book').get :id => book_id
+    assert !h['exists'], 'was not successfully deleted'
+
+    h = @index.docs('book').get :id => book_id2
+    assert_equal 'Tolkien', h['_source']['author']
+  end
+
+  it 'allows documents to be JSON strings' do
+    h = @index.bulk do |b|
+      b.index  '{"author":"pea53", "message":"just a test tweet"}', :_id => 1, :_type => 'tweet'
+      b.create '{"author":"John Scalzi", "title":"Old Mans War"}',  :_id => 1, :_type => 'book'
+    end
+    items = h['items']
+
+    assert_instance_of Fixnum, h['took']
+
+    assert_bulk_index h['items'].first
+    assert_bulk_create h['items'].last
+
+    @index.refresh
+
+    h = @index.docs('tweet').get :id => 1
+    assert_equal 'pea53', h['_source']['author']
+
+    h = @index.docs('book').get :id => 1
+    assert_equal 'John Scalzi', h['_source']['author']
+
+
+    h = @index.bulk do |b|
+      b.index '{"author":"Tolkien", "title":"The Silmarillion"}', :_id => 2, :_type => 'book'
+      b.delete :_id => 1, :_type => 'book'
+    end
+    items = h['items']
+
+    assert_bulk_index h['items'].first, 'expected to index a book'
+    assert_bulk_delete h['items'].last, 'expected to delete a book'
+
+    @index.refresh
+
+    h = @index.docs('book').get :id => 1
+    assert !h['exists'], 'was not successfully deleted'
+
+    h = @index.docs('book').get :id => 2
+    assert_equal 'Tolkien', h['_source']['author']
+  end
+
+  it 'executes a bulk API call when a request size is reached' do
+    ary = []
+    ary << @index.bulk(:request_size => 300) do |b|
+      2.times { |num|
+        document = {:_id => num, :_type => 'tweet', :author => 'pea53', :message => "tweet #{num} is a 100 character request"}
+        ary << b.index(document)
+      }
+      ary.compact!
+      assert_equal 0, ary.length
+
+      7.times { |num|
+        document = {:_id => num+2, :_type => 'tweet', :author => 'pea53', :message => "tweet #{num+2} is a 100 character request"}
+        ary << b.index(document)
+      }
+      ary.compact!
+      assert_equal 3, ary.length
+
+      document = {:_id => 10, :_type => 'tweet', :author => 'pea53', :message => "tweet 10 is a 102 character request"}
+      ary << b.index(document)
+    end
+    ary.compact!
+
+    assert_equal 4, ary.length
+    ary.each { |a| a['items'].each { |b| assert_bulk_index(b) } }
+
+    @index.refresh
+    h = @index.docs.search :q => '*:*', :search_type => 'count'
+
+    assert_equal 10, h['hits']['total']
+  end
+
+  it 'executes a bulk API call when an action count is reached' do
+    ary = []
+    ary << @index.bulk(:action_count => 3) do |b|
+      2.times { |num|
+        document = {:_id => num, :_type => 'tweet', :author => 'pea53', :message => "this is tweet number #{num}"}
+        ary << b.index(document)
+      }
+      ary.compact!
+      assert_equal 0, ary.length
+
+      7.times { |num|
+        document = {:_id => num+2, :_type => 'tweet', :author => 'pea53', :message => "this is tweet number #{num+2}"}
+        ary << b.index(document)
+      }
+      ary.compact!
+      assert_equal 3, ary.length
+
+      document = {:_id => 10, :_type => 'tweet', :author => 'pea53', :message => "this is tweet number 10"}
+      ary << b.index(document)
+    end
+    ary.compact!
+
+    assert_equal 4, ary.length
+    ary.each { |a| a['items'].each { |b| assert_bulk_index(b) } }
+
+    @index.refresh
+    h = @index.docs.search :q => '*:*', :search_type => 'count'
+
+    assert_equal 10, h['hits']['total']
+  end
+end