lusis-jruby-elasticsearch 0.0.13

data/lib/jruby-elasticsearch.rb

@@ -0,0 +1,5 @@
+require "jruby-elasticsearch/client"
+require "jruby-elasticsearch/bulkrequest"
+require "jruby-elasticsearch/bulkstream"
+
+module ElasticSearch; end;

data/lib/jruby-elasticsearch/actionlistener.rb

@@ -0,0 +1,48 @@
+require "java"
+require "jruby-elasticsearch/namespace"
+
+class ElasticSearch::ActionListener
+  include org.elasticsearch.action.ActionListener
+
+  def initialize
+    @failure_callbacks = []
+    @success_callbacks = []
+  end # def initialize
+
+  # Helper for registering callbacks.
+  # 'what' should be either :failure or :success
+  #
+  # You can register multiple callbacks if you wish.
+  # Callbacks are invoked in order of addition.
+  def on(what, &block)
+    case what
+    when :failure
+      @failure_callbacks << block
+    when :success
+      @success_callbacks << block
+    else
+      raise "Unknown event '#{what}' for #{self.class.name}"
+    end
+    return self
+  end # def on
+ 
+  # Conforming to Interface org.elasticsearch.action.ActionListener
+  def onFailure(exception)
+    if !@failure_callbacks.empty?
+      @failure_callbacks.each { |c| c.call(exception) }
+    else
+      # Default is no failure callbacks
+      raise exception
+    end
+  end # def onFailure
+
+  # Conforming to Interface org.elasticsearch.action.ActionListener
+  def onResponse(response)
+    if !@success_callbacks.empty?
+      @success_callbacks.each { |c| c.call(response) }
+    else
+      # Default if no success callbacks
+      puts "#{self.class.name}#onResponse => #{response.inspect} (#{self})"
+    end
+  end # def onResponse
+end # class ElasticSearch::ActionListener

data/lib/jruby-elasticsearch/bulkrequest.rb

@@ -0,0 +1,43 @@
+require "jruby-elasticsearch/namespace"
+require "jruby-elasticsearch/request"
+
+class ElasticSearch::BulkRequest < ElasticSearch::Request
+  # Create a new index request.
+  public
+  def initialize(client)
+    @client = client
+    @prep = @client.prepareBulk()
+    super()
+  end # def initialize
+
+  # Execute this index request.
+  # This call is asynchronous.
+  #
+  # If a block is given, register it for both failure and success.
+  def execute(&block)
+    use_callback(&block) if block_given?
+    action = @prep.execute(@handler)
+    return action
+  end # def execute
+
+  # Execute this index request synchronously
+  public
+  def execute!
+    return @prep.execute.actionGet()
+  end # def execute!
+
+  # Index a document.
+  public
+  def index(index, type, id=nil, data={})
+    req = org.elasticsearch.action.index.IndexRequest.new(index)
+    req.type(type) if type
+    req.id(id.to_s) if id
+    req.source(data)
+    @prep.add(req)
+  end
+
+  public
+  def <<(request)
+    @prep.add(request)
+  end # def <<
+end # def ElasticSearch::BulkRequest

data/lib/jruby-elasticsearch/bulkstream.rb

@@ -0,0 +1,80 @@
+require "jruby-elasticsearch/namespace"
+require "thread"
+
+class ElasticSearch::BulkStream
+  # Create a new bulk stream. This allows you to send
+  # index and other bulk events asynchronously and use
+  # the bulk api in ElasticSearch in a streaming way.
+  #
+  # The 'queue_size' is the maximum size of unflushed
+  # requests. If the queue reaches this size, new requests
+  # will block until there is room to move.
+  def initialize(client, queue_size=10, flush_interval=1)
+    @bulkthread = Thread.new { run } 
+    @client = client
+    @queue_size = queue_size
+    @queue = SizedQueue.new(@queue_size)
+    @flush_interval = flush_interval
+  end # def initialize
+
+  # See ElasticSearch::BulkRequest#index for arguments.
+  public
+  def index(*args)
+    # TODO(sissel): It's not clear I need to queue this up, I could just
+    # call BulkRequest#index() and when we have 10 or whatnot, flush, but
+    # Queue gives us a nice blocking mechanism anyway.
+    @queue << [:index, *args]
+  end # def index
+
+  # The stream runner. 
+  private
+  def run
+    # TODO(sissel): Make a way to shutdown this thread.
+    while true
+      requests = []
+      if @queue.size == @queue_size
+        # queue full, flush now.
+        flush
+      else
+        # Not full, so sleep and flush anyway.
+        sleep(@flush_interval)
+        flush
+      end
+
+      if @stop and @queue.size == 0
+        # Queue empty and it's time to stop.
+        break
+      end
+    end # while true
+  end # def run
+
+  # Stop the stream
+  public
+  def stop
+    @queue << nil
+    @stop = true
+  end # def stop
+
+  # Flush the queue right now. This will block until the
+  # bulk request has completed.
+  public
+  def flush
+    bulk = @client.bulk
+
+    flush_one = proc do
+      # block if no data.
+      method, *args = @queue.pop
+      return if args.nil? # probably we are now stopping.
+      bulk.send(method, *args)
+    end
+
+    flush_one.call
+
+    1.upto([@queue.size, @queue_size - 1].min) do
+      flush_one.call
+    end
+
+    # Block until this finishes
+    bulk.execute!
+  end # def flush
+end # class ElasticSearch::BulkStream

data/lib/jruby-elasticsearch/client.rb

@@ -0,0 +1,126 @@
+require "java"
+require "jruby-elasticsearch/namespace"
+require "jruby-elasticsearch/indexrequest"
+require "jruby-elasticsearch/searchrequest"
+
+class ElasticSearch::Client
+
+  # Creates a new ElasticSearch client.
+  #
+  # options:
+  # :type => [:local, :node] - :local will create a process-local
+  #   elasticsearch instances
+  # :host => "hostname" - the hostname to connect to.
+  # :port => 9200 - the port to connect to
+  # :cluster => "clustername" - the cluster name to use
+  def initialize(options={})
+    builder = org.elasticsearch.node.NodeBuilder.nodeBuilder
+    builder.client(true)
+
+    # The client doesn't need to serve http
+    builder.settings.put("http.enabled", false)
+
+    case options[:type]
+    when :local
+      builder.local(true)
+      @node = builder.node
+      @client = @node.client
+    when :transport
+      # TODO(sissel): Support transport client
+    else
+      # Use unicast discovery a host is given
+      if !options[:host].nil?
+        port = (options[:port] or "9300")
+        builder.settings.put("discovery.zen.ping.multicast.enabled", false)
+        builder.settings.put("discovery.zen.ping.unicast.hosts", "#{options[:host]}:#{port}")
+        #builder.settings.put("es.transport.tcp.port", port)
+      end
+
+      if options[:bind_host]
+        builder.settings.put('network.host', options[:bind_host])
+      end
+
+      if !options[:cluster].nil?
+        builder.clusterName(options[:cluster])
+      end
+      @node = builder.node
+      @client = @node.client
+    end
+
+  end # def initialize
+
+  # Get a new BulkRequest for sending multiple updates to elasticsearch in one
+  # request.
+  public
+  def bulk
+    return ElasticSearch::BulkRequest.new(@client)
+  end # def bulk
+
+  public
+  def bulkstream(queue_size=10, flush_interval=1)
+    return ElasticSearch::BulkStream.new(self, queue_size, flush_interval)
+  end # def bulk
+
+  # Index a new document
+  #
+  # args:
+  #   index: the index name
+  #   type: the type name
+  #   id: (optional) the id of the document
+  #   data: (optional) the data for this document
+  #   &block: (optional) optional block for using the DSL to add data
+  #
+  # Returns an ElasticSearch::IndexRequest instance.
+  #
+  # Example w/ DSL:
+  #
+  #     request = client.index("foo", "logs") do
+  #       filename "/var/log/message"
+  #       mesage "hello world"
+  #       timestamp 123456
+  #     end
+  #
+  #     request.execute!
+  def index(index, type, id=nil, data={}, &block)
+    # Permit 'id' being omitted entirely.
+    # Thus a call call: index("foo", "bar", somehash) is valid.
+    if id.is_a?(Hash)
+      data = id
+      id = nil
+    end
+
+    indexreq = ElasticSearch::IndexRequest.new(@client, index, type, id, data)
+    if block_given?
+      indexreq.instance_eval(&block)
+    end
+    return indexreq
+  end # def index
+
+  # Search for data.
+  # If a block is given, it is passed to SearchRequest#with so you can
+  # more easily configure the search, like so:
+  #
+  #   search = client.search("foo") do
+  #     query("*")
+  #     histogram("field", 1000)
+  #   end
+  #
+  #   The context of the block is of the SearchRequest object.
+  public
+  def search(&block)
+    searchreq = ElasticSearch::SearchRequest.new(@client)
+    if block_given?
+      searchreq.with(&block)
+    end
+    return searchreq
+  end # def search
+
+  def cluster
+    return @client.admin.cluster
+  end
+
+  def node
+    return @client.admin.cluster
+  end
+end # class ElasticSearch::Client
+

data/lib/jruby-elasticsearch/indexrequest.rb

@@ -0,0 +1,47 @@
+require "jruby-elasticsearch/namespace"
+require "jruby-elasticsearch/request"
+
+class ElasticSearch::IndexRequest < ElasticSearch::Request
+  # Create a new index request.
+  def initialize(client, index, type, id=nil, data={})
+    @client = client
+    @index = index
+    @type = type
+    @id = id
+    @data = data
+
+    # This should silence jruby warnings for 'multiple java methods for prepareIndex'
+    if id.nil?
+      @prep = @client.prepareIndex(index, type)
+    else
+      @prep = @client.prepareIndex(index, type, id)
+    end
+    super()
+  end
+
+  # Execute this index request.
+  # This call is asynchronous.
+  #
+  # If a block is given, register it for both failure and success.
+  def execute(&block)
+    @prep.setSource(@data)
+    use_callback(&block) if block_given?
+
+    action = @prep.execute(@handler)
+    return action
+  end
+
+  # Execute this index request synchronously
+  def execute!
+    @prep.setSource(@data)
+    return @prep.execute.actionGet()
+  end
+
+  # DSL helper.
+  # TODO(sissel): Move this away to a DSL module.
+  def method_missing(*args)
+    key, value = args
+    puts "Adding: #{key}: #{value.inspect}"
+    @data[key.to_s] = value
+  end
+end

data/lib/jruby-elasticsearch/namespace.rb

@@ -0,0 +1,4 @@
+
+module ElasticSearch
+# empty for now
+end

data/lib/jruby-elasticsearch/request.rb

@@ -0,0 +1,27 @@
+require "jruby-elasticsearch/namespace"
+require "jruby-elasticsearch/actionlistener"
+
+class ElasticSearch::Request
+  # Create a new index request.
+  def initialize
+    @handler = ElasticSearch::ActionListener.new
+  end
+
+  # See ElasticSearch::ActionListener#on
+  def on(event, &block)
+    #puts "Event[#{event}] => #{block} (#{@handler})"
+    @handler.on(event, &block)
+    return self
+  end
+
+  # Execute this index request.
+  # This call is asynchronous.
+  #
+  # If a block is given, register it for both failure and success.
+  def use_callback(&block)
+    if block_given?
+      on(:failure, &block)
+      on(:success, &block)
+    end
+  end
+end

data/lib/jruby-elasticsearch/searchrequest.rb

@@ -0,0 +1,138 @@
+require "jruby-elasticsearch/namespace"
+require "jruby-elasticsearch/request"
+
+class ElasticSearch::SearchRequest < ElasticSearch::Request
+  begin
+    QueryStringQueryBuilder = org.elasticsearch.index.query.xcontent.QueryStringQueryBuilder
+  rescue NameError
+    # The 'xcontent' namespace was removed in elasticsearch 0.17.0
+    QueryStringQueryBuilder = org.elasticsearch.index.query.QueryStringQueryBuilder
+  end
+
+  # Create a new index request.
+  public
+  def initialize(client)
+    @client = client
+    # Try the 0.19 API
+    begin
+      @prep = org.elasticsearch.action.search.SearchRequestBuilder.new(@client)
+    rescue NameError
+      # Okay so maybe the pre-0.19 API works?
+      @prep = org.elasticsearch.client.action.search.SearchRequestBuilder.new(@client)
+    end
+    @indeces = []
+    super()
+  end # def initialize
+
+  public
+  def with(&block)
+    instance_eval(&block)
+    return self
+  end # def with
+
+  public
+  def index(index_name)
+    @indeces << index_name
+  end
+
+  # Execute this search request.
+  # This call is asynchronous.
+  #
+  # If a block is given, register it for both failure and success.
+  #
+  # On success, callback will receive a
+  # org.elasticsearch.action.search.SearchResponse
+  public
+  def execute(&block)
+    use_callback(&block) if block_given?
+    @prep.setIndices(@indeces.to_java(:String))
+    action = @prep.execute(@handler)
+    return action
+  end # def execute
+
+  # Execute this index request synchronously
+  # Returns an org.elasticsearch.action.search.SearchResponse
+  public
+  def execute!
+    @prep.setIndices(@indeces.to_java(:String))
+    return @prep.execute.actionGet()
+  end # def execute!
+
+  public
+  def sort(field, order)
+    case order
+      when :asc
+        order_val = org.elasticsearch.search.sort.SortOrder::ASC
+      when :desc
+        order_val = org.elasticsearch.search.sort.SortOrder::DESC
+      else
+        raise "Invalid sort order '#{order.inspect}'"
+    end
+    @prep.addSort(field, order_val)
+    return self
+  end # def sort
+
+  public
+  def query(query_string, default_operator=:and)
+    # TODO(sissel): allow doing other queries and such.
+    qbuilder = QueryStringQueryBuilder.new(query_string)
+
+    operator = QueryStringQueryBuilder::Operator
+    case default_operator
+      when :and
+        qbuilder.defaultOperator(operator::AND)
+      when :or
+        qbuilder.defaultOperator(operator::OR)
+      else
+        raise "Unknown default operator '#{default_operator.inspect}'"
+    end
+
+    @prep.setQuery(qbuilder)
+    return self
+  end # def query
+
+  # Add a histogram facet to this query. Can be invoked multiple times.
+  public
+  def histogram(field, interval, name=nil)
+    if name.nil?
+      # TODO(sissel): How do we expose the name of the histogram?
+      name = "#{field}_#{interval}"
+    end
+    # TODO(sissel): Support 'global' ?
+    builder = org.elasticsearch.search.facet.histogram.HistogramFacetBuilder.new(name)
+    builder.field(field)
+    builder.interval(interval)
+    @prep.addFacet(builder)
+    return self
+  end # def histogram
+
+  public
+  def terms(field, name=nil)
+    if name.nil?
+      # TODO(sissel): How do we expose the name of the histogram?
+      name = field
+    end
+    # TODO(sissel): Support 'global' ?
+    builder = org.elasticsearch.search.facet.terms.TermsFacetBuilder.new(name)
+    builder.field(field)
+    @prep.addFacet(builder)
+    return self
+  end # def terms
+
+  public
+  def size(s)
+    @prep.setSize(s)
+    return self
+  end
+  alias :count :size
+  alias :limit :size
+
+  public
+  def from(from)
+    @prep.setFrom(from)
+    return self
+  end
+  alias :offset :from
+  alias :offset :from
+
+end # class ElasticSearch::SearchRequest

data/test/test_integration.rb

@@ -0,0 +1,108 @@
+require "test/unit"
+
+class TestElasticSearch < Test::Unit::TestCase
+  def setup
+    # Require all the elasticsearch libs
+    raise "Please set ELASTICSEARCH_HOME" if ENV['ELASTICSEARCH_HOME'].nil?
+
+    dir = File.join(ENV["ELASTICSEARCH_HOME"], "lib")
+    if !File.directory?(dir)
+      raise "ELASTICSEARCH_HOME set, but #{dir} doesn't exist"
+    end
+
+    Dir.glob(File.join(dir, "*.jar")).each do |jar|
+      require jar
+    end
+
+    $:.unshift(File.join(File.dirname(__FILE__), "..", "lib"))
+    require "jruby-elasticsearch"
+
+    # Start a local elasticsearch node
+    builder = org.elasticsearch.node.NodeBuilder.nodeBuilder
+    builder.local(true)
+    @elasticsearch = builder.node
+    @elasticsearch.start
+
+    # Create a client which will find a nearby elasticsearch cluster using
+    # the default discovery mechanism.
+    @client = ElasticSearch::Client.new({:type => :local})
+  end
+
+  def test_index_asynchronously
+    data = { "fizzle" => "dazzle", "pants" => "off" }
+    req = @client.index("twitter", "tweet", data)
+
+    # Set up async callbacks
+    done = false
+    req.on(:success) do |response|
+      assert_not_nil response
+      done = true
+    end.on(:failure) do |exception|
+      raise exception
+      done = true
+    end
+
+    # Execute it, but do it asynchronously.
+    req.execute
+
+    # Wait until we are done.
+    while !done
+      sleep 1
+    end
+  end
+
+  def test_bulk_index_asynchronously
+    data = { "fizzle" => "dazzle", "pants" => "off" }
+    bulk = @client.bulk
+    bulk.index("twitter", "tweet1", data)
+    bulk.index("twitter", "tweet2")
+
+    # Set up async callbacks
+    done = false
+    bulk.on(:success) do |response|
+      assert_not_nil response
+      done = true
+    end.on(:failure) do |exception|
+      raise exception
+      done = true
+    end
+
+    # Execute it, but do it asynchronously.
+    bulk.execute
+
+    # Wait until we are done.
+    while !done
+      sleep 1
+    end
+  end
+
+  def test_bulk_stream_synchronous
+    stream = @client.bulkstream(10)
+    tries = 10
+    entries = 30
+    1.upto(entries) do |i|
+      stream.index("hello", "world", { "foo" => "bar", "i" => i })
+    end
+    stream.stop
+
+    found = false
+    1.upto(tries) do
+      search = @client.search do
+        index "hello"
+        query "*"
+      end
+      # Results is an org.elasticsearch.action.search.SearchResponse
+      results = search.execute!
+      count = results.hits.totalHits
+
+      if count == entries
+        # assert for good measure
+        found = true
+        break
+      end
+      sleep 0.2
+    end # try a bunch to find our results
+
+    assert(found, "Search results were not found.")
+  end # def test_bulk_stream_synchronous
+end # class TestElasticSearch

metadata

@@ -0,0 +1,65 @@
+--- !ruby/object:Gem::Specification 
+name: lusis-jruby-elasticsearch
+version: !ruby/object:Gem::Version 
+  prerelease: 
+  version: 0.0.13
+platform: ruby
+authors: 
+  - Jordan Sissel
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2012-05-01 00:00:00 Z
+dependencies: []
+
+description: ...
+email: 
+  - jls@semicomplete.com
+executables: []
+
+extensions: []
+
+extra_rdoc_files: []
+
+files: 
+  - lib/jruby-elasticsearch.rb
+  - lib/jruby-elasticsearch/actionlistener.rb
+  - lib/jruby-elasticsearch/bulkrequest.rb
+  - lib/jruby-elasticsearch/bulkstream.rb
+  - lib/jruby-elasticsearch/client.rb
+  - lib/jruby-elasticsearch/indexrequest.rb
+  - lib/jruby-elasticsearch/namespace.rb
+  - lib/jruby-elasticsearch/request.rb
+  - lib/jruby-elasticsearch/searchrequest.rb
+  - test/test_integration.rb
+homepage: https://github.com/jordansissel/jruby-elasticsearch
+licenses: 
+  - Apache License (2.0)
+post_install_message: 
+rdoc_options: []
+
+require_paths: 
+  - lib
+  - lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: "0"
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: "0"
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.8.15
+signing_key: 
+specification_version: 3
+summary: JRuby API for ElasticSearch using the native ES Java API
+test_files: []
+