scalastic 0.4.0 → 0.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 5eaea5d5ea27529e7fd71ac835aa12a20fb4f6f7
-  data.tar.gz: ab107d4bc5e24cd69d3fe535d540957e4c4e83e5
+  metadata.gz: 8c137698c420bc7867a4dbf8caf041372c7c0ba5
+  data.tar.gz: d1a06b6c6fbf723fe018ba57a450b6cf446d1455
 SHA512:
-  metadata.gz: 2203b940e31523fd84ec2dee701ab3fc5ef46dc9007104592672b6e04dd0515c034ea0e1db249b21159dfc7d70d8c2c074a70fa94320a65ee3d8d02fbada7b2c
-  data.tar.gz: 80604212d8a574a5b94d208eb6cc9e363e6ad95131acce8dd733dde60168e8e0d4269e0cadbda5ac8e7a899d4e49dc4ba35b7b3de54384aa2a2e611b48abb464
+  metadata.gz: 141ecd5e0ae057f91fb67a374e58dae1a9dee830fc8697560506458ea2aa509b24fac08d34210a7453056a091519aa62cee7b1a0a99b0ac46debe7afd13856d8
+  data.tar.gz: aee055ffa89bfe275214bae37f02feddb0366dfc17e465fc366723f9095d61d7ae24c5ed19852bb0dcb831261c91ad401736703dc6938b6d34a3ad3ba9683939

data/README.md

@@ -238,6 +238,23 @@ actual_hits = p.search['hits']['hits']
 raise "Unexpected results!: #{actual_hits}" unless actual_hits == expected_hits
 ```
 
+### Customization
+Scalastic lets you choose prefix for partition aliases (the default value is `scalastic`):
+```ruby
+partitions.config.partition_prefix = 'my_partition'
+```
+This makes scalastic create aliases 'my_partition_123_index' and 'my_partition_123_search' for a partition 123.
+
+Also, you can customize the field that should be used for storing partition ids: 
+```ruby
+partitions.config.partition_selector = 'my_partition_id'
+```
+OR
+```ruby
+partitions.config.partition_selector = 'metadata.partition_id'
+```
+(note that the period in the partition selector indicates a nested field inside an object)
+
 ### Notes
 * Indices must be *prepared* before they can be used by Scalastic by calling "prepare" on the partitions client; doing so will create critical field mappings. Each index must be prepared only once. 
 * All hash keys in arguments must be symbols; using anything else may result in unexpected behavior.

data/Rakefile

@@ -1,5 +1,7 @@
 require 'bundler/gem_tasks'
 require 'rspec/core/rake_task'
+import 'regression/regression.rake'
 RSpec::Core::RakeTask.new(:spec)
 
 task :default => :spec
+

data/lib/elasticsearch/transport_client.rb

@@ -2,7 +2,7 @@ module Elasticsearch
   module Transport
     class Client
       def partitions
-        @partitions_client ||= Scalastic::PartitionsClient.new(self, Scalastic::Config.default)
+        @partitions_client ||= Scalastic::PartitionsClient.new(self)
       end
     end
   end

data/lib/scalastic/config.rb

@@ -2,14 +2,16 @@ module Scalastic
   class Config
     attr_reader(:partition_prefix)
     attr_reader(:partition_selector)
+    attr_reader(:partition_selector_type)
 
     def self.default
       @default ||= new
     end
 
     def initialize
-      @partition_prefix = "scalastic"
-      @partition_selector = :scalastic_partition_id
+      @partition_prefix = 'scalastic'
+      @partition_selector = 'scalastic_partition_id'
+      @partition_selector_type = 'long'
     end
 
     def index_endpoint(partition_id)
@@ -25,6 +27,28 @@ module Scalastic
       m && m[1].to_i
     end
 
+    def partition_prefix=(value)
+      raise(ArgumentError, 'Empty partition prefix') if value.nil? || value.empty?
+      @partition_prefix = value
+    end
+
+    def partition_selector=(value)
+      raise(ArgumentError, 'Empty partition selector') if value.nil? || value.empty?
+      @partition_selector = value
+    end
+
+    def partition_selector_type=(value)
+      value = value.to_s
+      raise(ArgumentError, "Unsupported selector type: #{value}. Supported types are: (string, long)") unless %w(string long).include?(value)
+      @partition_selector_type = value
+    end
+
+    def partition_selector_mapping
+      parts = partition_selector.to_s.split('.').reverse
+      field = parts.shift
+      parts.reduce(field => {type: partition_selector_type}){|acc, p| {p => {type: 'object', properties: acc}}}
+    end
+
     private
 
     def partition_regex

data/lib/scalastic/partition.rb

@@ -1,4 +1,5 @@
 require 'scalastic/es_actions_generator'
+require 'scalastic/partition_selector'
 
 module Scalastic
   class Partition
@@ -39,8 +40,8 @@ module Scalastic
     end
 
     def index(args)
-      args = {body: {}}.merge(args)
-      args[:body][config.partition_selector.to_sym] = id
+      args[:body] ||= {}
+      selector.apply_to(args[:body])
       args = args.merge(index: config.index_endpoint(id))
       es_client.index(args)
     end
@@ -58,8 +59,6 @@ module Scalastic
 
     def bulk(args)
       body = args.clone[:body] || raise(ArgumentError, 'Missing required argument :body')
-      index = config.index_endpoint(id)
-      selector = config.partition_selector
 
       new_ops = body.map{|entry| [operation_name(entry), entry]}.reduce([]){|acc, op| acc << [op.first, update_entry(acc, *op)]; acc}
       args[:body] = new_ops.map{|_op, entry| entry}
@@ -93,25 +92,12 @@ module Scalastic
       if (operation)
         op_data = entry[operation]
         op_data[:_index] = config.index_endpoint(id)
-        if (op_data[:data])
-          if (operation == :update)
-            op_data[:data][:doc] ||= {}
-            op_data[:data][:doc][config.partition_selector] = id
-          else
-            op_data[:data][config.partition_selector] = id
-          end
-        end
+        selector.apply_to(op_data[:data]) if op_data.has_key?(:data) && [:index, :create].include?(operation)
       else
         parent = acc.last
         # A previous record must be create/index/update/delete
         raise(ArgumentError, "Unexpected entry: #{entry}") unless parent && parent.first
-
-        if (parent.first == :update)
-          entry[:doc] ||= {}
-          entry[:doc][config.partition_selector] = id
-        else
-          entry[config.partition_selector] = id
-        end
+        selector.apply_to(entry) if [:index, :create].include?(parent.first)
       end
       entry
     end
@@ -119,5 +105,9 @@ module Scalastic
     def delete_op(hit)
       {delete: {_index: hit['_index'], _type: hit['_type'], _id: hit['_id']}}
     end
+
+    def selector
+      @selector ||= PartitionSelector.new(config.partition_selector, id)
+    end
   end
 end

data/lib/scalastic/partition_selector.rb

@@ -0,0 +1,14 @@
+module Scalastic
+  class PartitionSelector
+    def initialize(full_field_name, id)
+      @objects = full_field_name.split('.').map{|p| p.to_sym}
+      @field = @objects.pop
+      @id = id
+    end
+
+    def apply_to(document_body)
+      @objects.reduce(document_body){|body, obj| body[obj] ||= {}}[@field] = @id
+      document_body
+    end
+  end
+end

data/lib/scalastic/partitions_client.rb

@@ -6,9 +6,9 @@ module Scalastic
     attr_reader(:es_client)
     attr_reader(:config)
 
-    def initialize(es_client, config = Config.default)
-      raise(ArgumentError, "ES client is nil") if es_client.nil?
-      raise(ArgumentError, "Config is nil") if config.nil?
+    def initialize(es_client, config = Config.default.dup)
+      raise(ArgumentError, 'ES client is nil') if es_client.nil?
+      raise(ArgumentError, 'Config is nil') if config.nil?
       @es_client = es_client
       @config = config
     end
@@ -23,8 +23,8 @@ module Scalastic
     end
 
     def delete(args = {})
-      id = args[:id] || raise(ArgumentError, "Missing required argument :id")
-      pairs = es_client.indices.get_aliases.map{|i, d| d["aliases"].keys.select{|a| config.get_partition_id(a) == id}.map{|a| [i, a]}}.flatten(1)
+      id = args[:id] || raise(ArgumentError, 'Missing required argument :id')
+      pairs = es_client.indices.get_aliases.map{|i, d| d['aliases'].keys.select{|a| config.get_partition_id(a) == id}.map{|a| [i, a]}}.flatten(1)
       unless pairs.any?
         #TODO: log a warning
         return
@@ -39,15 +39,15 @@ module Scalastic
 
     def to_a
       aliases = es_client.indices.get_aliases
-      partition_ids = aliases.map{|_, data| data["aliases"].keys}.flatten.map{|a| config.get_partition_id(a)}.compact.uniq
+      partition_ids = aliases.map{|_, data| data['aliases'].keys}.flatten.map{|a| config.get_partition_id(a)}.compact.uniq
       partition_ids.map{|id| Partition.new(es_client, config, id)}
     end
 
     def prepare_index(args)
-      index = args[:index] || raise(ArgumentError, "Missing required argument :index")
-      mapping = {properties: {config.partition_selector => {type: "long"}}}
-      es_client.indices.put_mapping(index: index, type: "_default_", body: {"_default_" => mapping})
-      es_client.indices.put_mapping(index: index, type: "scalastic", body: {"scalastic" => mapping})
+      index = args[:index] || raise(ArgumentError, 'Missing required argument :index')
+      mapping = {properties: config.partition_selector_mapping}
+      es_client.indices.put_mapping(index: index, type: '_default_', body: {'_default_' => mapping})
+      es_client.indices.put_mapping(index: index, type: 'scalastic', body: {'scalastic' => mapping})
     end
   end
 end

data/lib/scalastic/version.rb

@@ -1,3 +1,3 @@
 module Scalastic
-  VERSION = "0.4.0"
+  VERSION = "0.5.0"
 end

data/lib/scalastic.rb

@@ -5,5 +5,4 @@ require 'scalastic/partitions_client'
 require 'elasticsearch/transport_client'
 
 module Scalastic
-  # Your code goes here...
 end

data/regression/regression.rake

@@ -0,0 +1,16 @@
+require_relative 'regression_tests'
+
+namespace :regression do
+  task :run do
+    RegressionTests.to_a.shuffle.each do |t|
+      puts "Running #{t.name}"
+      t.cleanup
+      t.run
+      t.cleanup
+    end
+  end
+
+  task :cleanup do
+    RegressionTests.each{|t| t.cleanup}
+  end
+end

data/regression/regression_tests/bulk_operations.rb

@@ -0,0 +1,71 @@
+module RegressionTests
+  module BulkOperations
+    extend self
+
+    def cleanup
+      client = Elasticsearch::Client.new
+      client.indices.delete index: 'bulk_operations' if client.indices.exists? index: 'bulk_operations'
+    end
+
+    def run
+      client = Elasticsearch::Client.new
+      partitions = client.partitions
+
+      client.indices.create(index: 'bulk_operations')
+      partitions.prepare_index(index: 'bulk_operations')
+
+      partition = partitions.create(index: 'bulk_operations', id: 1)
+      sleep 1.5
+
+      partition.bulk(body: [
+        {index: {_type: 'test', _id: 1, data: {subject: 'test1'}}},
+        {create: {_type: 'test', _id: 2, data: {subject: 'test2'}}}
+      ])
+
+      partition.bulk(body: [
+        {index: {_type: 'test', _id: 3}},
+        {subject: 'test3'},
+        {create: {_type: 'test', _id: 4}},
+        {subject: 'test4'}
+      ])
+
+      partition.bulk(body: [
+        {update: {_type: 'test', _id: 1, data: {doc: {body: 'Document 1'}}}},
+        {update: {_type: 'test', _id: 2, data: {doc: {body: 'Document 2'}}}}
+      ])
+
+      partition.bulk(body: [
+        {update: {_type: 'test', _id: 3}},
+        {doc: {body: 'Document 3'}},
+        {update: {_type: 'test', _id: 4}},
+        {doc: {body: 'Document 4'}}
+      ])
+
+      sleep 1.5
+
+      hits = partition.search['hits']['hits'].sort{|h1, h2| h1['_id'].to_i <=> h2['_id'].to_i}
+      raise "Expected 4 hits, got #{hits.size}" unless hits.size == 4
+
+      expected_hits = [
+        {'_index' => 'bulk_operations', '_type' => 'test', '_id' => '1', '_score' => 1.0, '_source' => {'subject' => 'test1', 'body' => 'Document 1', 'scalastic_partition_id' => 1}},
+        {'_index' => 'bulk_operations', '_type' => 'test', '_id' => '2', '_score' => 1.0, '_source' => {'subject' => 'test2', 'body' => 'Document 2', 'scalastic_partition_id' => 1}},
+        {'_index' => 'bulk_operations', '_type' => 'test', '_id' => '3', '_score' => 1.0, '_source' => {'subject' => 'test3', 'body' => 'Document 3', 'scalastic_partition_id' => 1}},
+        {'_index' => 'bulk_operations', '_type' => 'test', '_id' => '4', '_score' => 1.0, '_source' => {'subject' => 'test4', 'body' => 'Document 4', 'scalastic_partition_id' => 1}},
+      ]
+
+      raise 'Unexpected results' unless hits == expected_hits
+
+      res = partition.bulk(body: [
+        {delete: {_type: 'test', _id: 1}},
+        {delete: {_type: 'test', _id: 2}},
+        {delete: {_type: 'test', _id: 3}},
+        {delete: {_type: 'test', _id: 4}},
+      ])
+
+      sleep 1.5
+
+      count = partition.search(search_type: 'count')['hits']['total']
+      raise 'Some documents were not removed' unless count == 0
+    end
+  end
+end

data/regression/regression_tests/create_partition.rb

@@ -0,0 +1,25 @@
+module RegressionTests
+  module CreatePartition
+    extend self
+
+    def cleanup
+      client = Elasticsearch::Client.new
+      client.indices.delete index: 'create_partition' if client.indices.exists? index: 'create_partition'
+    end
+
+    def run
+      # Connect to Elasticsearch and get the partitions client
+      es_client = Elasticsearch::Client.new
+      partitions = es_client.partitions
+      
+      # Create an index for the test.
+      es_client.indices.create index: 'create_partition'
+      partitions.prepare_index index: 'create_partition'   # Needs to be called only once per index
+
+      # Create a partition
+      partition = partitions.create index: 'create_partition', id: 1
+      sleep 1.5
+      raise 'Partition was not created' unless partition.exists?
+    end
+  end
+end

data/regression/regression_tests/custom_partition_prefix.rb

@@ -0,0 +1,28 @@
+module RegressionTests
+  module CustomPartitionPrefix
+    extend self
+
+    def cleanup
+      client = Elasticsearch::Client.new
+      client.indices.delete index: 'custom_partition_prefix' if client.indices.exists? index: 'custom_partition_prefix'
+    end
+
+    def run
+      client = Elasticsearch::Client.new
+      partitions = client.partitions
+
+      partitions.config.partition_prefix = 'custom'
+
+      client.indices.create index: 'custom_partition_prefix'
+      partitions.prepare_index index: 'custom_partition_prefix'     # Must be called only once per index
+
+      partition = partitions.create index: 'custom_partition_prefix', id: 1
+      sleep 1.5
+
+      # Are aliases there? 
+      aliases = client.indices.get_aliases index: 'custom_partition_prefix'
+      expected_aliases = {"custom_partition_prefix"=>{"aliases"=>{"custom_1_index"=>{}, "custom_1_search"=>{"filter"=>{"term"=>{"scalastic_partition_id"=>1}}}}}}
+      raise "Expected: #{expected_aliases}; got: #{aliases}" unless expected_aliases == aliases
+    end
+  end
+end

data/regression/regression_tests/delete_by_query.rb

@@ -0,0 +1,31 @@
+module RegressionTests
+  module DeleteByQuery
+    extend self
+
+    def cleanup
+      client = Elasticsearch::Client.new
+      client.indices.delete index: 'delete_by_query' if client.indices.exists? index: 'delete_by_query'
+    end
+
+    def run
+      client = Elasticsearch::Client.new
+      partitions = client.partitions
+
+      client.indices.create(index: 'delete_by_query')
+      partitions.prepare_index(index: 'delete_by_query')
+
+      p = partitions.create(index: 'delete_by_query', id: 1)
+      p.index(id: 1, type: 'test')
+      p.index(id: 2, type: 'test')
+      p.index(id: 3, type: 'test')
+      sleep 1.5
+
+      p.delete_by_query(body:{query:{terms:{_id: [1,3]}}})
+      sleep 1.5
+
+      expected_hits = [{'_index' => 'delete_by_query', '_type' => 'test', '_id' => '2', '_score' => 1.0, '_source' => {'scalastic_partition_id' => 1}}]
+      actual_hits = p.search['hits']['hits']
+      raise "Unexpected results!: #{actual_hits}" unless actual_hits == expected_hits
+    end
+  end
+end

data/regression/regression_tests/delete_partition.rb

@@ -0,0 +1,30 @@
+module RegressionTests
+  module DeletePartition
+    extend self
+    
+    def cleanup
+      client = Elasticsearch::Client.new
+      client.indices.delete index: 'delete_partition' if client.indices.exists? index: 'delete_partition'
+    end
+
+    def run
+      # Connect to Elasticsearch and create an index
+      client = Elasticsearch::Client.new
+      partitions = client.partitions
+      client.indices.create index: 'delete_partition'
+      partitions.prepare_index index: 'delete_partition'
+
+      # Create partitions
+      partitions.create index: 'delete_partition', id: 1
+      partitions.create index: 'delete_partition', id: 2
+      partitions.create index: 'delete_partition', id: 3
+      sleep 1.5
+
+      # Delete one of the partitions
+      partitions.delete id: 2
+      sleep 1.5
+
+      raise 'Partition still exists' if partitions[2].exists?
+    end
+  end
+end

data/regression/regression_tests/document_get.rb

@@ -0,0 +1,34 @@
+module RegressionTests
+  module DocumentGet
+    extend self
+
+    def cleanup
+      client = Elasticsearch::Client.new
+      client.indices.delete index: 'document_get' if client.indices.exists? index: 'document_get'
+    end
+
+    def run
+      client = Elasticsearch::Client.new
+      client.indices.create(index: 'document_get')
+      client.partitions.prepare_index(index: 'document_get')
+
+      p = client.partitions.create(id: 1, index: 'document_get')
+      p.index(id: 1, type: 'test', body: {title: 'Test'})
+
+      sleep 1.5
+
+      res = p.get(id: 1, type: 'test')['_source']
+      expected = {'title' => 'Test', 'scalastic_partition_id' => 1}
+      raise "Expected: #{expected}, got: #{res}" unless res == expected
+
+      p = client.partitions.create(id: 2, index: 'document_get', routing: 12345)
+      p.index(id: 2, type: 'test', body: {title: 'Routing test'})
+
+      sleep 1.5
+      
+      res = p.get(id: 2, type: 'test')['_source']
+      expected = {'title' => 'Routing test', 'scalastic_partition_id' => 2}
+      raise "Expected: #{expected}, got: #{res}" unless res == expected
+    end
+  end
+end

data/regression/regression_tests/extend_partition.rb

@@ -0,0 +1,29 @@
+module RegressionTests
+  module ExtendPartition
+    extend self
+
+    def cleanup
+      client = Elasticsearch::Client.new
+      %w(extend_partition_1 extend_partition_2).each do |i|
+        client.indices.delete index: i if client.indices.exists? index: i
+      end
+    end
+
+    def run
+      # Connect to Elasticsearch and set up indices
+      client = Elasticsearch::Client.new
+      partitions = client.partitions
+      client.indices.create index: 'extend_partition_1'
+      partitions.prepare_index index: 'extend_partition_1'
+      client.indices.create index: 'extend_partition_2'
+      partitions.prepare_index index: 'extend_partition_2'
+
+      # Create a partition residing in extend_partition_1
+      partition = partitions.create(index: 'extend_partition_1', id: 1)
+
+      # Extend partition to index extend_partition_2. Now search will be performed in both indices, but 
+      # all new documents will be indexex into extend_partition_2.
+      partition.extend_to(index: 'extend_partition_2')
+    end
+  end
+end

data/regression/regression_tests/list_partitions.rb

@@ -0,0 +1,35 @@
+module RegressionTests
+  module ListPartitions
+    extend self
+
+    def cleanup
+      client = Elasticsearch::Client.new
+      client.indices.delete index: 'list_partitions' if client.indices.exists? index: 'list_partitions'
+    end
+
+    def run
+      # Set everything up
+      client = Elasticsearch::Client.new
+      client.indices.create index: 'list_partitions'
+      partitions = client.partitions
+      partitions.prepare_index index: 'list_partitions'    # Must be called once per each index
+
+      sleep 1.5
+
+      # Create a couple of partitions
+      partitions.create index: 'list_partitions', id: 1
+      partitions.create index: 'list_partitions', id: 2
+      partitions.create index: 'list_partitions', id: 3
+
+      partitions[1].index type: 'test', body: {title: 'In partition 1'}
+      partitions[2].index type: 'test', body: {title: 'In partition 2'}
+      partitions[3].index type: 'test', body: {title: 'In partition 3'}
+
+      sleep 1.5
+
+      # List all partitions
+      ids = partitions.to_a.map{|p| p.id}
+      raise "Unexpected partitions: #{ids}" unless ids.sort == [1,2,3]
+    end
+  end
+end

data/regression/regression_tests/nested_selector.rb

@@ -0,0 +1,43 @@
+module RegressionTests
+  module NestedSelector
+    extend self
+
+    def cleanup
+      client = Elasticsearch::Client.new
+      client.indices.delete index: 'nested_selector' if client.indices.exists? index: 'nested_selector'
+    end
+
+    def run
+      client = Elasticsearch::Client.new
+      partitions = client.partitions
+
+      # Create an index for testing
+      client.indices.create index: 'nested_selector'
+
+      # Prepare the environment
+      partitions.config.partition_selector = 'parent.child.partition_id'
+      partitions.prepare_index index: 'nested_selector'     # Must be called only once for the index
+
+      # Indexing should work
+      partition = partitions.create index: 'nested_selector', id: 1
+      partition.index type: 'document', id: 1, body: {title: 'Test', content: 'This is a test'}
+      
+      sleep 1.5
+
+      # Searching should work, too
+      results = partition.search(type: 'document', body: {query: {match_all: {}}})
+      count = results['hits']['total']
+      raise "Expected 1 result, got #{count}" unless count == 1
+
+      hit = results['hits']['hits'].first['_source']
+      expected = {'title'=>'Test', 'content'=>'This is a test', 'parent'=>{'child'=>{'partition_id'=>1}}}
+      raise "Expected: #{expected}, got: #{hit}" unless expected == hit
+
+      # Deleting should work
+      partition.delete type: 'document', id: 1
+      sleep 1.5
+      count = partition.search(search_type: 'count', body: {query: {match_all: {}}})['hits']['total']
+      raise "Expected 0 records, got #{count}" unless count == 0
+    end
+  end
+end

data/regression/regression_tests/nested_selector_bulk.rb

@@ -0,0 +1,85 @@
+module RegressionTests
+  module NestedSelectorBulk
+    extend self
+
+    def cleanup
+      client = Elasticsearch::Client.new
+      client.indices.delete index: 'nested_selector_bulk' if client.indices.exists? index: 'nested_selector_bulk'
+    end
+
+    def run
+      client = Elasticsearch::Client.new
+
+      # Set up test environment
+      client.indices.create index: 'nested_selector_bulk'
+      partitions = client.partitions
+      partitions.config.partition_selector = 'parent.child.partition_id'
+      partitions.prepare_index index: 'nested_selector_bulk'          # Set up field mapping
+
+      # Create partition
+      partition = partitions.create index: 'nested_selector_bulk', id: 1
+
+      # Index documents
+      partition.bulk body: [
+        {index: {_type: 'test', _id: 1, data: {subject: 'Test 1'}}},
+        {create: {_type: 'test', _id: 2, data: {subject: 'Test 2'}}},
+      ]
+
+      partition.bulk body: [
+        {index: {_type: 'test', _id: 3}},
+        {subject: 'Test 3'},
+        {create: {_type: 'test', _id: 4}},
+        {subject: 'Test 4'}
+      ]
+
+      sleep 1.5
+
+      hits = partition.search(body: {query: {match_all: {}}})['hits']['hits'].sort{|h1, h2| h1['_id'] <=> h2['_id']}
+      expected_hits = [
+        {'_index'=>'nested_selector_bulk', '_type' => 'test', '_id' => '1', '_score' => 1.0, '_source' => {'subject' => 'Test 1', 'parent' => {'child' => {'partition_id' => 1}}}},
+        {'_index'=>'nested_selector_bulk', '_type' => 'test', '_id' => '2', '_score' => 1.0, '_source' => {'subject' => 'Test 2', 'parent' => {'child' => {'partition_id' => 1}}}},
+        {'_index'=>'nested_selector_bulk', '_type' => 'test', '_id' => '3', '_score' => 1.0, '_source' => {'subject' => 'Test 3', 'parent' => {'child' => {'partition_id' => 1}}}},
+        {'_index'=>'nested_selector_bulk', '_type' => 'test', '_id' => '4', '_score' => 1.0, '_source' => {'subject' => 'Test 4', 'parent' => {'child' => {'partition_id' => 1}}}},
+      ]
+      raise "Expected: #{expected_hits}, got: #{hits}" unless expected_hits == hits
+
+      # Update documents
+      partition.bulk body: [
+        {update: {_type: 'test', _id: 1, data: {doc: {subject: 'Test 1a'}}}},
+        {update: {_type: 'test', _id: 2, data: {doc: {subject: 'Test 2a'}}}},
+        {update: {_type: 'test', _id: 3}},
+        {doc: {subject: 'Test 3a'}}
+      ]
+
+      partition.bulk body: [
+        {update: {_type: 'test', _id: 3}},
+        {doc: {subject: 'Test 3a'}},
+        {update: {_type: 'test', _id: 4}},
+        {doc: {subject: 'Test 4a'}}
+      ]
+
+      sleep 1.5
+
+      hits = partition.search(body: {query: {match_all: {}}})['hits']['hits'].sort{|h1, h2| h1['_id'] <=> h2['_id']}
+      expected_hits = [
+        {'_index'=>'nested_selector_bulk', '_type' => 'test', '_id' => '1', '_score' => 1.0, '_source' => {'subject' => 'Test 1a', 'parent' => {'child' => {'partition_id' => 1}}}},
+        {'_index'=>'nested_selector_bulk', '_type' => 'test', '_id' => '2', '_score' => 1.0, '_source' => {'subject' => 'Test 2a', 'parent' => {'child' => {'partition_id' => 1}}}},
+        {'_index'=>'nested_selector_bulk', '_type' => 'test', '_id' => '3', '_score' => 1.0, '_source' => {'subject' => 'Test 3a', 'parent' => {'child' => {'partition_id' => 1}}}},
+        {'_index'=>'nested_selector_bulk', '_type' => 'test', '_id' => '4', '_score' => 1.0, '_source' => {'subject' => 'Test 4a', 'parent' => {'child' => {'partition_id' => 1}}}},
+      ]
+      raise "Expected: #{expected_hits}, got: #{hits}" unless expected_hits == hits
+
+      partition.bulk body: [
+        {delete: {_type: 'test', _id: 1}},
+        {delete: {_type: 'test', _id: 2}},
+        {delete: {_type: 'test', _id: 3}},
+        {delete: {_type: 'test', _id: 4}},
+      ]
+
+      sleep 1.5
+
+      hits = partition.search['hits']['hits']
+      raise "Expected no hits, got: #{hits}" unless hits.empty?
+    end
+  end
+end

data/regression/regression_tests/partition_operations.rb

@@ -0,0 +1,45 @@
+module RegressionTests
+  module PartitionOperations
+    extend self
+
+    def cleanup
+      client = Elasticsearch::Client.new
+      client.indices.delete index: 'partition_operations' if client.indices.exists? index: 'partition_operations'
+    end
+
+    def run
+      client = Elasticsearch::Client.new
+      partitions = client.partitions
+
+      client.indices.create index: 'partition_operations'
+      partitions.prepare_index index: 'partition_operations'
+      partition1 = partitions.create(index: 'partition_operations', id: 1)
+      partition2 = partitions.create(index: 'partition_operations', id: 2)
+
+      partition1.index id: 1, type: 'document', body: {subject: 'Subject 1'}
+      partition1.index id: 2, type: 'document', body: {subject: 'Subject 2'}
+
+      sleep 1.5
+
+      # Partition 2 should have no documents
+      count = partition2.search(search_type: 'count', body: {query: {match_all: {}}})['hits']['total']
+      raise 'Partition 2 is not empty!' unless count == 0
+
+      # Partiton 1 should contain everything we just indexed
+      hits = partition1.search(type: 'document', body: {query:{match_all: {}}})['hits']['hits']
+      raise "Expected 2 documents, got #{hits.size}" unless hits.size == 2
+      h1 = hits.find{|h| h['_id'].to_i == 1}
+      raise 'Document 1 cannot be found' unless h1
+      partition_id = h1['_source']['scalastic_partition_id']
+      raise "Expected: 1; got: #{partition_id}" unless partition_id == 1
+
+      # Now delete something from partition 1
+      partition1.delete type: 'document', id: 1
+
+      sleep 1.5
+
+      count = partition1.search(search_type: 'count', body: {query: {match_all: {}}})['hits']['total']
+      raise "Expected 1 document, got #{count}" unless count == 1
+    end
+  end
+end

data/regression/regression_tests/string_selector_field.rb

@@ -0,0 +1,32 @@
+module RegressionTests
+  module StringSelectorField
+    extend self
+
+    def cleanup
+      client = Elasticsearch::Client.new
+      client.indices.delete index: 'string_selector_field' if client.indices.exists? index: 'string_selector_field'
+    end
+
+    def run
+      client = Elasticsearch::Client.new
+      partitions = client.partitions
+
+      client.indices.create index: 'string_selector_field'
+
+      partitions.config.partition_selector_type = 'string'
+      partitions.prepare_index index: 'string_selector_field'
+
+      partition = partitions.create index: 'string_selector_field', id: 'favorites'
+      partition.index type: 'license', body: {title: 'This is a test'}
+
+      sleep 1.5
+
+      hits = partition.search(type: 'license')['hits']['hits']
+      raise "Expected 1 result, got #{hits.size}" unless hits.size == 1
+      
+      expected = {'title'=>'This is a test', 'scalastic_partition_id'=>'favorites'}
+      actual = hits.first['_source']
+      raise "Expected: #{expected}, got: #{actual}" unless expected == actual
+    end
+  end
+end

data/regression/regression_tests.rb

@@ -0,0 +1,15 @@
+require 'scalastic'
+require 'elasticsearch'
+
+module RegressionTests
+  include Enumerable
+  extend self
+
+  def each(&block)
+    Dir.glob('./regression/regression_tests/**.rb').each do |l|
+      load l
+    end
+
+    RegressionTests.constants.map{|c| RegressionTests.const_get(c)}.select{|c| c.is_a?(Module) && c.respond_to?(:run) && c.respond_to?(:cleanup)}.each(&block)
+  end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: scalastic
 version: !ruby/object:Gem::Version
-  version: 0.4.0
+  version: 0.5.0
 platform: ruby
 authors:
 - Aliaksei Baturytski
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2016-02-29 00:00:00.000000000 Z
+date: 2016-04-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -100,8 +100,23 @@ files:
 - lib/scalastic/config.rb
 - lib/scalastic/es_actions_generator.rb
 - lib/scalastic/partition.rb
+- lib/scalastic/partition_selector.rb
 - lib/scalastic/partitions_client.rb
 - lib/scalastic/version.rb
+- regression/regression.rake
+- regression/regression_tests.rb
+- regression/regression_tests/bulk_operations.rb
+- regression/regression_tests/create_partition.rb
+- regression/regression_tests/custom_partition_prefix.rb
+- regression/regression_tests/delete_by_query.rb
+- regression/regression_tests/delete_partition.rb
+- regression/regression_tests/document_get.rb
+- regression/regression_tests/extend_partition.rb
+- regression/regression_tests/list_partitions.rb
+- regression/regression_tests/nested_selector.rb
+- regression/regression_tests/nested_selector_bulk.rb
+- regression/regression_tests/partition_operations.rb
+- regression/regression_tests/string_selector_field.rb
 - scalastic.gemspec
 homepage: https://github.com/aliakb/scalastic
 licenses: