jruby-elasticsearch 0.0.11 → 0.0.12

data/lib/jruby-elasticsearch/bulkrequest.rb

@@ -3,11 +3,12 @@ require "jruby-elasticsearch/request"
 
 class ElasticSearch::BulkRequest < ElasticSearch::Request
   # Create a new index request.
+  public
   def initialize(client)
     @client = client
     @prep = @client.prepareBulk()
     super()
-  end
+  end # def initialize
 
   # Execute this index request.
   # This call is asynchronous.
@@ -17,13 +18,16 @@ class ElasticSearch::BulkRequest < ElasticSearch::Request
     use_callback(&block) if block_given?
     action = @prep.execute(@handler)
     return action
-  end
+  end # def execute
 
   # Execute this index request synchronously
+  public
   def execute!
     return @prep.execute.actionGet()
-  end
+  end # def execute!
 
+  # Index a document.
+  public
   def index(index, type, id=nil, data={})
     req = org.elasticsearch.action.index.IndexRequest.new(index)
     req.type(type) if type
@@ -32,6 +36,7 @@ class ElasticSearch::BulkRequest < ElasticSearch::Request
     @prep.add(req)
   end
 
+  public
   def <<(request)
     @prep.add(request)
   end # def <<

data/lib/jruby-elasticsearch/bulkstream.rb

@@ -1,7 +1,80 @@
 require "jruby-elasticsearch/namespace"
+require "thread"
 
 class ElasticSearch::BulkStream
-  def initialize
-    @bulkthread = Thread.new
-  end
-end
+  # Create a new bulk stream. This allows you to send
+  # index and other bulk events asynchronously and use
+  # the bulk api in ElasticSearch in a streaming way.
+  #
+  # The 'queue_size' is the maximum size of unflushed
+  # requests. If the queue reaches this size, new requests
+  # will block until there is room to move.
+  def initialize(client, queue_size=10, flush_interval=1)
+    @bulkthread = Thread.new { run } 
+    @client = client
+    @queue_size = queue_size
+    @queue = SizedQueue.new(@queue_size)
+    @flush_interval = flush_interval
+  end # def initialize
+
+  # See ElasticSearch::BulkRequest#index for arguments.
+  public
+  def index(*args)
+    # TODO(sissel): It's not clear I need to queue this up, I could just
+    # call BulkRequest#index() and when we have 10 or whatnot, flush, but
+    # Queue gives us a nice blocking mechanism anyway.
+    @queue << [:index, *args]
+  end # def index
+
+  # The stream runner. 
+  private
+  def run
+    # TODO(sissel): Make a way to shutdown this thread.
+    while true
+      requests = []
+      if @queue.size == @queue_size
+        # queue full, flush now.
+        flush
+      else
+        # Not full, so sleep and flush anyway.
+        sleep(@flush_interval)
+        flush
+      end
+
+      if @stop and @queue.size == 0
+        # Queue empty and it's time to stop.
+        break
+      end
+    end # while true
+  end # def run
+
+  # Stop the stream
+  public
+  def stop
+    @queue << nil
+    @stop = true
+  end # def stop
+
+  # Flush the queue right now. This will block until the
+  # bulk request has completed.
+  public
+  def flush
+    bulk = @client.bulk
+
+    flush_one = proc do
+      # block if no data.
+      method, *args = @queue.pop
+      return if args.nil? # probably we are now stopping.
+      bulk.send(method, *args)
+    end
+
+    flush_one.call
+
+    1.upto([@queue.size, @queue_size - 1].min) do
+      flush_one.call
+    end
+
+    # Block until this finishes
+    bulk.execute!
+  end # def flush
+end # class ElasticSearch::BulkStream

data/lib/jruby-elasticsearch/client.rb

@@ -53,7 +53,12 @@ class ElasticSearch::Client
   # request.
   public
   def bulk
-    ElasticSearch::BulkRequest.new(@client)
+    return ElasticSearch::BulkRequest.new(@client)
+  end # def bulk
+
+  public
+  def bulkstream(queue_size=10, flush_interval=1)
+    return ElasticSearch::BulkStream.new(self, queue_size, flush_interval)
   end # def bulk
 
   # Index a new document

data/lib/jruby-elasticsearch.rb

@@ -1,9 +1,5 @@
 require "jruby-elasticsearch/client"
 require "jruby-elasticsearch/bulkrequest"
-
-#class Proc
-  #include java.lang.Runnable
-  #alias_method :run, :call
-#end
+require "jruby-elasticsearch/bulkstream"
 
 module ElasticSearch; end;

data/test/test_integration.rb

@@ -4,11 +4,17 @@ class TestElasticSearch < Test::Unit::TestCase
   def setup
     # Require all the elasticsearch libs
     raise "Please set ELASTICSEARCH_HOME" if ENV['ELASTICSEARCH_HOME'].nil?
-    Dir[File.join(ENV['ELASTICSEARCH_HOME'],"lib/*.jar")].each do |jar|
+
+    dir = File.join(ENV["ELASTICSEARCH_HOME"], "lib")
+    if !File.directory?(dir)
+      raise "ELASTICSEARCH_HOME set, but #{dir} doesn't exist"
+    end
+
+    Dir.glob(File.join(dir, "*.jar")).each do |jar|
       require jar
     end
 
-    $:.unshift("lib")
+    $:.unshift(File.join(File.dirname(__FILE__), "..", "lib"))
     require "jruby-elasticsearch"
 
     # Start a local elasticsearch node
@@ -69,4 +75,34 @@ class TestElasticSearch < Test::Unit::TestCase
       sleep 1
     end
   end
-end
+
+  def test_bulk_stream_synchronous
+    stream = @client.bulkstream(10)
+    tries = 10
+    entries = 30
+    1.upto(entries) do |i|
+      stream.index("hello", "world", { "foo" => "bar", "i" => i })
+    end
+    stream.stop
+
+    found = false
+    1.upto(tries) do
+      search = @client.search do
+        index "hello"
+        query "*"
+      end
+      # Results is an org.elasticsearch.action.search.SearchResponse
+      results = search.execute!
+      count = results.hits.totalHits
+
+      if count == entries
+        # assert for good measure
+        found = true
+        break
+      end
+      sleep 0.2
+    end # try a bunch to find our results
+
+    assert(found, "Search results were not found.")
+  end # def test_bulk_stream_synchronous
+end # class TestElasticSearch

metadata

@@ -2,7 +2,7 @@
 name: jruby-elasticsearch
 version: !ruby/object:Gem::Version 
   prerelease: 
-  version: 0.0.11
+  version: 0.0.12
 platform: ruby
 authors: 
   - Jordan Sissel
@@ -10,7 +10,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2011-10-11 00:00:00 -07:00
+date: 2011-10-14 00:00:00 -07:00
 default_executable: 
 dependencies: []