tire-erez 0.5.4
Sign up to get free protection for your applications and to get access to all the features.
- data/.gitignore +14 -0
- data/.travis.yml +32 -0
- data/.yardopts +1 -0
- data/Gemfile +10 -0
- data/MIT-LICENSE +20 -0
- data/README.markdown +775 -0
- data/Rakefile +51 -0
- data/examples/rails-application-template.rb +263 -0
- data/examples/tire-dsl.rb +932 -0
- data/lib/tire.rb +59 -0
- data/lib/tire/alias.rb +296 -0
- data/lib/tire/configuration.rb +38 -0
- data/lib/tire/count.rb +85 -0
- data/lib/tire/dsl.rb +114 -0
- data/lib/tire/http/client.rb +62 -0
- data/lib/tire/http/clients/curb.rb +61 -0
- data/lib/tire/http/clients/faraday.rb +71 -0
- data/lib/tire/http/response.rb +27 -0
- data/lib/tire/index.rb +443 -0
- data/lib/tire/logger.rb +60 -0
- data/lib/tire/model/callbacks.rb +40 -0
- data/lib/tire/model/import.rb +27 -0
- data/lib/tire/model/indexing.rb +134 -0
- data/lib/tire/model/naming.rb +100 -0
- data/lib/tire/model/percolate.rb +99 -0
- data/lib/tire/model/persistence.rb +72 -0
- data/lib/tire/model/persistence/attributes.rb +148 -0
- data/lib/tire/model/persistence/finders.rb +54 -0
- data/lib/tire/model/persistence/storage.rb +77 -0
- data/lib/tire/model/search.rb +322 -0
- data/lib/tire/multi_search.rb +263 -0
- data/lib/tire/results/collection.rb +156 -0
- data/lib/tire/results/item.rb +94 -0
- data/lib/tire/results/pagination.rb +68 -0
- data/lib/tire/rubyext/hash.rb +8 -0
- data/lib/tire/rubyext/ruby_1_8.rb +1 -0
- data/lib/tire/rubyext/symbol.rb +11 -0
- data/lib/tire/rubyext/uri_escape.rb +74 -0
- data/lib/tire/search.rb +211 -0
- data/lib/tire/search/facet.rb +81 -0
- data/lib/tire/search/filter.rb +28 -0
- data/lib/tire/search/highlight.rb +37 -0
- data/lib/tire/search/queries/match.rb +40 -0
- data/lib/tire/search/query.rb +250 -0
- data/lib/tire/search/scan.rb +114 -0
- data/lib/tire/search/script_field.rb +23 -0
- data/lib/tire/search/sort.rb +25 -0
- data/lib/tire/tasks.rb +138 -0
- data/lib/tire/utils.rb +17 -0
- data/lib/tire/version.rb +18 -0
- data/test/fixtures/articles/1.json +1 -0
- data/test/fixtures/articles/2.json +1 -0
- data/test/fixtures/articles/3.json +1 -0
- data/test/fixtures/articles/4.json +1 -0
- data/test/fixtures/articles/5.json +1 -0
- data/test/integration/active_model_indexing_test.rb +51 -0
- data/test/integration/active_model_searchable_test.rb +114 -0
- data/test/integration/active_record_searchable_test.rb +620 -0
- data/test/integration/boolean_queries_test.rb +43 -0
- data/test/integration/boosting_queries_test.rb +32 -0
- data/test/integration/bulk_test.rb +86 -0
- data/test/integration/count_test.rb +64 -0
- data/test/integration/custom_score_queries_test.rb +89 -0
- data/test/integration/dis_max_queries_test.rb +68 -0
- data/test/integration/dsl_search_test.rb +30 -0
- data/test/integration/explanation_test.rb +44 -0
- data/test/integration/facets_test.rb +311 -0
- data/test/integration/filtered_queries_test.rb +66 -0
- data/test/integration/filters_test.rb +75 -0
- data/test/integration/fuzzy_queries_test.rb +20 -0
- data/test/integration/highlight_test.rb +64 -0
- data/test/integration/index_aliases_test.rb +122 -0
- data/test/integration/index_mapping_test.rb +43 -0
- data/test/integration/index_store_test.rb +112 -0
- data/test/integration/index_update_document_test.rb +121 -0
- data/test/integration/match_query_test.rb +79 -0
- data/test/integration/mongoid_searchable_test.rb +309 -0
- data/test/integration/multi_search_test.rb +114 -0
- data/test/integration/nested_query_test.rb +135 -0
- data/test/integration/percolator_test.rb +111 -0
- data/test/integration/persistent_model_test.rb +205 -0
- data/test/integration/prefix_query_test.rb +43 -0
- data/test/integration/query_return_version_test.rb +70 -0
- data/test/integration/query_string_test.rb +52 -0
- data/test/integration/range_queries_test.rb +36 -0
- data/test/integration/reindex_test.rb +56 -0
- data/test/integration/results_test.rb +58 -0
- data/test/integration/scan_test.rb +56 -0
- data/test/integration/script_fields_test.rb +38 -0
- data/test/integration/sort_test.rb +52 -0
- data/test/integration/text_query_test.rb +39 -0
- data/test/models/active_model_article.rb +31 -0
- data/test/models/active_model_article_with_callbacks.rb +49 -0
- data/test/models/active_model_article_with_custom_document_type.rb +7 -0
- data/test/models/active_model_article_with_custom_index_name.rb +7 -0
- data/test/models/active_record_models.rb +131 -0
- data/test/models/article.rb +15 -0
- data/test/models/mongoid_models.rb +85 -0
- data/test/models/persistent_article.rb +11 -0
- data/test/models/persistent_article_in_index.rb +16 -0
- data/test/models/persistent_article_in_namespace.rb +12 -0
- data/test/models/persistent_article_with_casting.rb +28 -0
- data/test/models/persistent_article_with_defaults.rb +12 -0
- data/test/models/persistent_article_with_percolation.rb +5 -0
- data/test/models/persistent_articles_with_custom_index_name.rb +10 -0
- data/test/models/supermodel_article.rb +17 -0
- data/test/models/validated_model.rb +11 -0
- data/test/test_helper.rb +118 -0
- data/test/unit/active_model_lint_test.rb +17 -0
- data/test/unit/configuration_test.rb +84 -0
- data/test/unit/count_test.rb +67 -0
- data/test/unit/http_client_test.rb +79 -0
- data/test/unit/http_response_test.rb +49 -0
- data/test/unit/index_alias_test.rb +335 -0
- data/test/unit/index_test.rb +1098 -0
- data/test/unit/logger_test.rb +125 -0
- data/test/unit/model_callbacks_test.rb +116 -0
- data/test/unit/model_import_test.rb +75 -0
- data/test/unit/model_initialization_test.rb +31 -0
- data/test/unit/model_persistence_test.rb +548 -0
- data/test/unit/model_search_test.rb +964 -0
- data/test/unit/multi_search_test.rb +304 -0
- data/test/unit/results_collection_test.rb +372 -0
- data/test/unit/results_item_test.rb +173 -0
- data/test/unit/rubyext_test.rb +66 -0
- data/test/unit/search_facet_test.rb +186 -0
- data/test/unit/search_filter_test.rb +42 -0
- data/test/unit/search_highlight_test.rb +46 -0
- data/test/unit/search_query_test.rb +419 -0
- data/test/unit/search_scan_test.rb +113 -0
- data/test/unit/search_script_field_test.rb +26 -0
- data/test/unit/search_sort_test.rb +50 -0
- data/test/unit/search_test.rb +556 -0
- data/test/unit/tire_test.rb +144 -0
- data/tire.gemspec +83 -0
- metadata +586 -0
data/lib/tire/dsl.rb
ADDED
@@ -0,0 +1,114 @@
|
|
1
|
+
module Tire
|
2
|
+
module DSL
|
3
|
+
|
4
|
+
def configure(&block)
|
5
|
+
Configuration.class_eval(&block)
|
6
|
+
end
|
7
|
+
|
8
|
+
def search(indices=nil, options={}, &block)
|
9
|
+
if block_given?
|
10
|
+
Search::Search.new(indices, options, &block)
|
11
|
+
else
|
12
|
+
payload = case options
|
13
|
+
when Hash then
|
14
|
+
options
|
15
|
+
when String then
|
16
|
+
Tire.warn "Passing the payload as a JSON string in Tire.search has been deprecated, " +
|
17
|
+
"please use the block syntax or pass a plain Hash."
|
18
|
+
options
|
19
|
+
else raise ArgumentError, "Please pass a Ruby Hash or String with JSON"
|
20
|
+
end
|
21
|
+
unless options.empty?
|
22
|
+
Search::Search.new(indices, :payload => payload)
|
23
|
+
else
|
24
|
+
Search::Search.new(indices)
|
25
|
+
end
|
26
|
+
end
|
27
|
+
rescue Exception => error
|
28
|
+
STDERR.puts "[REQUEST FAILED] #{error.class} #{error.message rescue nil}\n"
|
29
|
+
raise
|
30
|
+
ensure
|
31
|
+
end
|
32
|
+
|
33
|
+
# Build and perform a [multi-search](http://elasticsearch.org/guide/reference/api/multi-search.html)
|
34
|
+
# request.
|
35
|
+
#
|
36
|
+
# s = Tire.multi_search 'clients' do
|
37
|
+
# search :names do
|
38
|
+
# query { match :name, 'carpenter' }
|
39
|
+
# end
|
40
|
+
# search :counts, search_type: 'count' do
|
41
|
+
# query { match [:name, :street, :occupation], 'carpenter' }
|
42
|
+
# end
|
43
|
+
# search :vip, index: 'clients-vip' do
|
44
|
+
# query { string "last_name:carpenter" }
|
45
|
+
# end
|
46
|
+
# search() { query {all} }
|
47
|
+
# end
|
48
|
+
#
|
49
|
+
# The DSL allows you to perform multiple searches and get corresponding results
|
50
|
+
# in a single HTTP request, saving network roundtrips.
|
51
|
+
#
|
52
|
+
# Use the `search` method in the block to define a search request with the
|
53
|
+
# regular Tire's DSL (`query`, `facet`, etc).
|
54
|
+
#
|
55
|
+
# You can pass options such as `search_type`, `routing`, etc.,
|
56
|
+
# as well as a different `index` and/or `type` to individual searches.
|
57
|
+
#
|
58
|
+
# You can give single searches names, to be able to refer to them later.
|
59
|
+
#
|
60
|
+
# The results are returned as an enumerable collection of {Tire::Results::Collection} instances.
|
61
|
+
#
|
62
|
+
# You may simply iterate over them with `each`:
|
63
|
+
#
|
64
|
+
# s.results.each do |results|
|
65
|
+
# puts results.map(&:name)
|
66
|
+
# end
|
67
|
+
#
|
68
|
+
# To iterate over named results, use the `each_pair` method:
|
69
|
+
#
|
70
|
+
# s.results.each_pair do |name,results|
|
71
|
+
# puts "Search #{name} got #{results.size} results"
|
72
|
+
# end
|
73
|
+
#
|
74
|
+
# You can get a specific named result:
|
75
|
+
#
|
76
|
+
# search.results[:vip]
|
77
|
+
#
|
78
|
+
# You can mix & match named and non-named searches in the definition; the non-named
|
79
|
+
# searches will be zero-based numbered, so you can refer to them:
|
80
|
+
#
|
81
|
+
# search.results[3] # Results for the last query
|
82
|
+
#
|
83
|
+
# To log the multi-search request, use the standard `to_curl` method (or set up a logger):
|
84
|
+
#
|
85
|
+
# print search.to_curl
|
86
|
+
#
|
87
|
+
def multi_search(indices=nil, options={}, &block)
|
88
|
+
Search::Multi::Search.new(indices, options, &block)
|
89
|
+
rescue Exception => error
|
90
|
+
STDERR.puts "[REQUEST FAILED] #{error.class} #{error.message rescue nil}\n"
|
91
|
+
raise
|
92
|
+
ensure
|
93
|
+
end
|
94
|
+
alias :multisearch :multi_search
|
95
|
+
alias :msearch :multi_search
|
96
|
+
|
97
|
+
def count(indices=nil, options={}, &block)
|
98
|
+
Search::Count.new(indices, options, &block).value
|
99
|
+
end
|
100
|
+
|
101
|
+
def index(name, &block)
|
102
|
+
Index.new(name, &block)
|
103
|
+
end
|
104
|
+
|
105
|
+
def scan(names, options={}, &block)
|
106
|
+
Search::Scan.new(names, options, &block)
|
107
|
+
end
|
108
|
+
|
109
|
+
def aliases
|
110
|
+
Alias.all
|
111
|
+
end
|
112
|
+
|
113
|
+
end
|
114
|
+
end
|
@@ -0,0 +1,62 @@
|
|
1
|
+
module Tire
|
2
|
+
|
3
|
+
module HTTP
|
4
|
+
|
5
|
+
module Client
|
6
|
+
|
7
|
+
class RestClient
|
8
|
+
ConnectionExceptions = [::RestClient::ServerBrokeConnection, ::RestClient::RequestTimeout]
|
9
|
+
|
10
|
+
def self.get(url, data=nil)
|
11
|
+
perform ::RestClient::Request.new(:method => :get, :url => url, :payload => data).execute
|
12
|
+
rescue *ConnectionExceptions
|
13
|
+
raise
|
14
|
+
rescue ::RestClient::Exception => e
|
15
|
+
Response.new e.http_body, e.http_code
|
16
|
+
end
|
17
|
+
|
18
|
+
def self.post(url, data)
|
19
|
+
perform ::RestClient.post(url, data)
|
20
|
+
rescue *ConnectionExceptions
|
21
|
+
raise
|
22
|
+
rescue ::RestClient::Exception => e
|
23
|
+
Response.new e.http_body, e.http_code
|
24
|
+
end
|
25
|
+
|
26
|
+
def self.put(url, data)
|
27
|
+
perform ::RestClient.put(url, data)
|
28
|
+
rescue *ConnectionExceptions
|
29
|
+
raise
|
30
|
+
rescue ::RestClient::Exception => e
|
31
|
+
Response.new e.http_body, e.http_code
|
32
|
+
end
|
33
|
+
|
34
|
+
def self.delete(url)
|
35
|
+
perform ::RestClient.delete(url)
|
36
|
+
rescue *ConnectionExceptions
|
37
|
+
raise
|
38
|
+
rescue ::RestClient::Exception => e
|
39
|
+
Response.new e.http_body, e.http_code
|
40
|
+
end
|
41
|
+
|
42
|
+
def self.head(url)
|
43
|
+
perform ::RestClient.head(url)
|
44
|
+
rescue *ConnectionExceptions
|
45
|
+
raise
|
46
|
+
rescue ::RestClient::Exception => e
|
47
|
+
Response.new e.http_body, e.http_code
|
48
|
+
end
|
49
|
+
|
50
|
+
private
|
51
|
+
|
52
|
+
def self.perform(response)
|
53
|
+
Response.new response.body, response.code, response.headers
|
54
|
+
end
|
55
|
+
|
56
|
+
end
|
57
|
+
|
58
|
+
end
|
59
|
+
|
60
|
+
end
|
61
|
+
|
62
|
+
end
|
@@ -0,0 +1,61 @@
|
|
1
|
+
require 'curb'
|
2
|
+
|
3
|
+
module Tire
|
4
|
+
|
5
|
+
module HTTP
|
6
|
+
|
7
|
+
module Client
|
8
|
+
|
9
|
+
class Curb
|
10
|
+
@client = ::Curl::Easy.new
|
11
|
+
@client.resolve_mode = :ipv4
|
12
|
+
|
13
|
+
# @client.verbose = true
|
14
|
+
|
15
|
+
def self.get(url, data=nil)
|
16
|
+
@client.url = url
|
17
|
+
|
18
|
+
# FIXME: Curb cannot post bodies with GET requests?
|
19
|
+
# Roy Fielding seems to approve:
|
20
|
+
# <http://tech.groups.yahoo.com/group/rest-discuss/message/9962>
|
21
|
+
if data
|
22
|
+
@client.post_body = data
|
23
|
+
@client.http_post
|
24
|
+
else
|
25
|
+
@client.http_get
|
26
|
+
end
|
27
|
+
Response.new @client.body_str, @client.response_code
|
28
|
+
end
|
29
|
+
|
30
|
+
def self.post(url, data)
|
31
|
+
@client.url = url
|
32
|
+
@client.post_body = data
|
33
|
+
@client.http_post
|
34
|
+
Response.new @client.body_str, @client.response_code
|
35
|
+
end
|
36
|
+
|
37
|
+
def self.put(url, data)
|
38
|
+
@client.url = url
|
39
|
+
@client.http_put data
|
40
|
+
Response.new @client.body_str, @client.response_code
|
41
|
+
end
|
42
|
+
|
43
|
+
def self.delete(url)
|
44
|
+
@client.url = url
|
45
|
+
@client.http_delete
|
46
|
+
Response.new @client.body_str, @client.response_code
|
47
|
+
end
|
48
|
+
|
49
|
+
def self.head(url)
|
50
|
+
@client.url = url
|
51
|
+
@client.http_head
|
52
|
+
Response.new @client.body_str, @client.response_code
|
53
|
+
end
|
54
|
+
|
55
|
+
end
|
56
|
+
|
57
|
+
end
|
58
|
+
|
59
|
+
end
|
60
|
+
|
61
|
+
end
|
@@ -0,0 +1,71 @@
|
|
1
|
+
require 'faraday'
|
2
|
+
|
3
|
+
# A Faraday-based HTTP client, which allows you to choose a HTTP client.
|
4
|
+
#
|
5
|
+
# See <https://github.com/technoweenie/faraday/tree/master/lib/faraday/adapter>
|
6
|
+
#
|
7
|
+
# NOTE: Tire will switch to Faraday for the HTTP abstraction layer. This client is a temporary solution.
|
8
|
+
#
|
9
|
+
# Example:
|
10
|
+
# --------
|
11
|
+
#
|
12
|
+
# require 'typhoeus'
|
13
|
+
# require 'tire/http/clients/faraday'
|
14
|
+
#
|
15
|
+
# Tire.configure do |config|
|
16
|
+
#
|
17
|
+
# # Unless specified, tire will use Faraday.default_adapter and no middleware
|
18
|
+
# Tire::HTTP::Client::Faraday.faraday_middleware = Proc.new do |builder|
|
19
|
+
# builder.adapter :typhoeus
|
20
|
+
# end
|
21
|
+
#
|
22
|
+
# config.client(Tire::HTTP::Client::Faraday)
|
23
|
+
#
|
24
|
+
# end
|
25
|
+
#
|
26
|
+
#
|
27
|
+
module Tire
|
28
|
+
module HTTP
|
29
|
+
module Client
|
30
|
+
class Faraday
|
31
|
+
|
32
|
+
# Default middleware stack.
|
33
|
+
DEFAULT_MIDDLEWARE = Proc.new do |builder|
|
34
|
+
builder.adapter ::Faraday.default_adapter
|
35
|
+
end
|
36
|
+
|
37
|
+
class << self
|
38
|
+
# A customized stack of Faraday middleware that will be used to make each request.
|
39
|
+
attr_accessor :faraday_middleware
|
40
|
+
|
41
|
+
def get(url, data = nil)
|
42
|
+
request(:get, url, data)
|
43
|
+
end
|
44
|
+
|
45
|
+
def post(url, data)
|
46
|
+
request(:post, url, data)
|
47
|
+
end
|
48
|
+
|
49
|
+
def put(url, data)
|
50
|
+
request(:put, url, data)
|
51
|
+
end
|
52
|
+
|
53
|
+
def delete(url, data = nil)
|
54
|
+
request(:delete, url, data)
|
55
|
+
end
|
56
|
+
|
57
|
+
def head(url)
|
58
|
+
request(:head, url)
|
59
|
+
end
|
60
|
+
|
61
|
+
private
|
62
|
+
def request(method, url, data = nil)
|
63
|
+
conn = ::Faraday.new( &(faraday_middleware || DEFAULT_MIDDLEWARE) )
|
64
|
+
response = conn.run_request(method, url, data, nil)
|
65
|
+
Response.new(response.body, response.status, response.headers)
|
66
|
+
end
|
67
|
+
end
|
68
|
+
end
|
69
|
+
end
|
70
|
+
end
|
71
|
+
end
|
@@ -0,0 +1,27 @@
|
|
1
|
+
module Tire
|
2
|
+
|
3
|
+
module HTTP
|
4
|
+
|
5
|
+
class Response
|
6
|
+
attr_reader :body, :code, :headers
|
7
|
+
|
8
|
+
def initialize(body, code, headers={})
|
9
|
+
@body, @code, @headers = body, code.to_i, headers
|
10
|
+
end
|
11
|
+
|
12
|
+
def success?
|
13
|
+
code > 0 && code < 400
|
14
|
+
end
|
15
|
+
|
16
|
+
def failure?
|
17
|
+
! success?
|
18
|
+
end
|
19
|
+
|
20
|
+
def to_s
|
21
|
+
[code, body].join(' : ')
|
22
|
+
end
|
23
|
+
end
|
24
|
+
|
25
|
+
end
|
26
|
+
|
27
|
+
end
|
data/lib/tire/index.rb
ADDED
@@ -0,0 +1,443 @@
|
|
1
|
+
module Tire
|
2
|
+
class Index
|
3
|
+
|
4
|
+
attr_reader :name, :response
|
5
|
+
|
6
|
+
def initialize(name, &block)
|
7
|
+
@name = name
|
8
|
+
block.arity < 1 ? instance_eval(&block) : block.call(self) if block_given?
|
9
|
+
end
|
10
|
+
|
11
|
+
def url
|
12
|
+
"#{Configuration.url}/#{@name}"
|
13
|
+
end
|
14
|
+
|
15
|
+
def exists?
|
16
|
+
@response = Configuration.client.head("#{url}")
|
17
|
+
@response.success?
|
18
|
+
|
19
|
+
ensure
|
20
|
+
curl = %Q|curl -I "#{url}"|
|
21
|
+
logged('HEAD', curl)
|
22
|
+
end
|
23
|
+
|
24
|
+
def delete
|
25
|
+
@response = Configuration.client.delete url
|
26
|
+
@response.success?
|
27
|
+
|
28
|
+
ensure
|
29
|
+
curl = %Q|curl -X DELETE #{url}|
|
30
|
+
logged('DELETE', curl)
|
31
|
+
end
|
32
|
+
|
33
|
+
def create(options={})
|
34
|
+
@options = options
|
35
|
+
@response = Configuration.client.post url, MultiJson.encode(options)
|
36
|
+
@response.success? ? @response : false
|
37
|
+
|
38
|
+
ensure
|
39
|
+
curl = %Q|curl -X POST #{url} -d '#{MultiJson.encode(options, :pretty => Configuration.pretty)}'|
|
40
|
+
logged('CREATE', curl)
|
41
|
+
end
|
42
|
+
|
43
|
+
def add_alias(alias_name, configuration={})
|
44
|
+
Alias.create(configuration.merge( :name => alias_name, :index => @name ) )
|
45
|
+
end
|
46
|
+
|
47
|
+
def remove_alias(alias_name)
|
48
|
+
Alias.find(alias_name) { |a| a.indices.delete @name }.save
|
49
|
+
end
|
50
|
+
|
51
|
+
def aliases(alias_name=nil)
|
52
|
+
alias_name ? Alias.all(@name).select { |a| a.name == alias_name }.first : Alias.all(@name)
|
53
|
+
end
|
54
|
+
|
55
|
+
def mapping
|
56
|
+
@response = Configuration.client.get("#{url}/_mapping")
|
57
|
+
MultiJson.decode(@response.body)[@name]
|
58
|
+
end
|
59
|
+
|
60
|
+
def settings
|
61
|
+
@response = Configuration.client.get("#{url}/_settings")
|
62
|
+
MultiJson.decode(@response.body)[@name]['settings']
|
63
|
+
end
|
64
|
+
|
65
|
+
def store(*args)
|
66
|
+
document, options = args
|
67
|
+
|
68
|
+
id = get_id_from_document(document)
|
69
|
+
type = get_type_from_document(document)
|
70
|
+
document = convert_document_to_json(document)
|
71
|
+
|
72
|
+
options ||= {}
|
73
|
+
params = {}
|
74
|
+
|
75
|
+
if options[:percolate]
|
76
|
+
params[:percolate] = options[:percolate]
|
77
|
+
params[:percolate] = "*" if params[:percolate] === true
|
78
|
+
end
|
79
|
+
|
80
|
+
params[:parent] = options[:parent] if options[:parent]
|
81
|
+
params[:routing] = options[:routing] if options[:routing]
|
82
|
+
params[:replication] = options[:replication] if options[:replication]
|
83
|
+
|
84
|
+
params_encoded = params.empty? ? '' : "?#{params.to_param}"
|
85
|
+
|
86
|
+
url = id ? "#{self.url}/#{type}/#{id}#{params_encoded}" : "#{self.url}/#{type}/#{params_encoded}"
|
87
|
+
|
88
|
+
@response = Configuration.client.post url, document
|
89
|
+
MultiJson.decode(@response.body)
|
90
|
+
ensure
|
91
|
+
curl = %Q|curl -X POST "#{url}" -d '#{document}'|
|
92
|
+
logged([type, id].join('/'), curl)
|
93
|
+
end
|
94
|
+
|
95
|
+
# Performs a [bulk](http://www.elasticsearch.org/guide/reference/api/bulk.html) request
|
96
|
+
#
|
97
|
+
# @myindex.bulk :index, [ {id: 1, title: 'One'}, { id: 2, title: 'Two', _version: 3 } ], refresh: true
|
98
|
+
#
|
99
|
+
# Pass the action (`index`, `create`, `delete`) as the first argument, the collection of documents as
|
100
|
+
# the second argument, and URL parameters as the last option.
|
101
|
+
#
|
102
|
+
# Any _meta_ information contained in documents (such as `_routing` or `_parent`) is extracted
|
103
|
+
# and added to the "header" line.
|
104
|
+
#
|
105
|
+
# Shortcut methods `bulk_store`, `bulk_delete` and `bulk_create` are available.
|
106
|
+
#
|
107
|
+
def bulk(action, documents, options={})
|
108
|
+
return false if documents.empty?
|
109
|
+
|
110
|
+
# TODO: A more Ruby-like DSL notation should be supported:
|
111
|
+
#
|
112
|
+
# Tire.index('myindex').bulk do
|
113
|
+
# create id: 1, title: 'bar', _routing: 'abc'
|
114
|
+
# delete id: 1
|
115
|
+
# # ...
|
116
|
+
# end
|
117
|
+
|
118
|
+
payload = documents.map do |document|
|
119
|
+
type = get_type_from_document(document, :escape => false) # Do not URL-escape the _type
|
120
|
+
id = get_id_from_document(document)
|
121
|
+
|
122
|
+
if ENV['DEBUG']
|
123
|
+
STDERR.puts "[ERROR] Document #{document.inspect} does not have ID" unless id
|
124
|
+
end
|
125
|
+
|
126
|
+
header = { action.to_sym => { :_index => name, :_type => type, :_id => id } }
|
127
|
+
|
128
|
+
if document.respond_to?(:to_hash) && hash = document.to_hash
|
129
|
+
meta = {}
|
130
|
+
meta[:_version] = hash.delete(:_version)
|
131
|
+
meta[:_routing] = hash.delete(:_routing)
|
132
|
+
meta[:_percolate] = hash.delete(:_percolate)
|
133
|
+
meta[:_parent] = hash.delete(:_parent)
|
134
|
+
meta[:_timestamp] = hash.delete(:_timestamp)
|
135
|
+
meta[:_ttl] = hash.delete(:_ttl)
|
136
|
+
meta = meta.reject { |name,value| !value || value.empty? }
|
137
|
+
header[action.to_sym].update(meta)
|
138
|
+
end
|
139
|
+
|
140
|
+
output = []
|
141
|
+
output << MultiJson.encode(header)
|
142
|
+
output << convert_document_to_json(document) unless action.to_s == 'delete'
|
143
|
+
output.join("\n")
|
144
|
+
end
|
145
|
+
payload << ""
|
146
|
+
|
147
|
+
tries = 5
|
148
|
+
count = 0
|
149
|
+
|
150
|
+
begin
|
151
|
+
params = {}
|
152
|
+
params[:consistency] = options.delete(:consistency)
|
153
|
+
params[:refresh] = options.delete(:refresh)
|
154
|
+
params = params.reject { |name,value| !value }
|
155
|
+
params_encoded = params.empty? ? '' : "?#{params.to_param}"
|
156
|
+
|
157
|
+
@response = Configuration.client.post("#{url}/_bulk#{params_encoded}", payload.join("\n"))
|
158
|
+
raise RuntimeError, "#{@response.code} > #{@response.body}" if @response && @response.failure?
|
159
|
+
@response
|
160
|
+
rescue StandardError => error
|
161
|
+
if count < tries
|
162
|
+
count += 1
|
163
|
+
STDERR.puts "[ERROR] #{error.message}, retrying (#{count})..."
|
164
|
+
retry
|
165
|
+
else
|
166
|
+
STDERR.puts "[ERROR] Too many exceptions occured, giving up. The HTTP response was: #{error.message}"
|
167
|
+
raise if options[:raise]
|
168
|
+
end
|
169
|
+
|
170
|
+
ensure
|
171
|
+
curl = %Q|curl -X POST "#{url}/_bulk" --data-binary '{... data omitted ...}'|
|
172
|
+
logged('_bulk', curl)
|
173
|
+
end
|
174
|
+
|
175
|
+
end
|
176
|
+
|
177
|
+
def bulk_create(documents, options={})
|
178
|
+
bulk :create, documents, options
|
179
|
+
end
|
180
|
+
|
181
|
+
def bulk_store(documents, options={})
|
182
|
+
bulk :index, documents, options
|
183
|
+
end
|
184
|
+
|
185
|
+
def bulk_delete(documents, options={})
|
186
|
+
bulk :delete, documents, options
|
187
|
+
end
|
188
|
+
|
189
|
+
def import(klass_or_collection, options={})
|
190
|
+
case
|
191
|
+
when method = options.delete(:method)
|
192
|
+
options = {:page => 1, :per_page => 1000}.merge options
|
193
|
+
while documents = klass_or_collection.send(method.to_sym, options.merge(:page => options[:page])) \
|
194
|
+
and documents.to_a.length > 0
|
195
|
+
|
196
|
+
documents = yield documents if block_given?
|
197
|
+
|
198
|
+
bulk_store documents, options
|
199
|
+
options[:page] += 1
|
200
|
+
end
|
201
|
+
|
202
|
+
when klass_or_collection.respond_to?(:map)
|
203
|
+
documents = block_given? ? yield(klass_or_collection) : klass_or_collection
|
204
|
+
bulk_store documents, options
|
205
|
+
|
206
|
+
else
|
207
|
+
raise ArgumentError, "Please pass either an Enumerable compatible class, or a collection object" +
|
208
|
+
"with a method for fetching records in batches (such as 'paginate')"
|
209
|
+
end
|
210
|
+
end
|
211
|
+
|
212
|
+
def reindex(name, options={}, &block)
|
213
|
+
new_index = Index.new(name)
|
214
|
+
new_index.create(options) unless new_index.exists?
|
215
|
+
|
216
|
+
transform = options.delete(:transform)
|
217
|
+
|
218
|
+
Search::Scan.new(self.name, &block).each do |results|
|
219
|
+
|
220
|
+
documents = results.map do |document|
|
221
|
+
document = document.to_hash.except(:type, :_index, :_explanation, :_score, :_version, :highlight, :sort)
|
222
|
+
document = transform.call(document) if transform
|
223
|
+
document
|
224
|
+
end
|
225
|
+
|
226
|
+
new_index.bulk_store documents
|
227
|
+
end
|
228
|
+
end
|
229
|
+
|
230
|
+
def remove(*args)
|
231
|
+
if args.size > 1
|
232
|
+
type, document = args
|
233
|
+
type = Utils.escape(type)
|
234
|
+
id = get_id_from_document(document) || document
|
235
|
+
else
|
236
|
+
document = args.pop
|
237
|
+
type = get_type_from_document(document)
|
238
|
+
id = get_id_from_document(document) || document
|
239
|
+
end
|
240
|
+
raise ArgumentError, "Please pass a document ID" unless id
|
241
|
+
|
242
|
+
url = "#{self.url}/#{type}/#{id}"
|
243
|
+
result = Configuration.client.delete url
|
244
|
+
MultiJson.decode(result.body) if result.success?
|
245
|
+
|
246
|
+
ensure
|
247
|
+
curl = %Q|curl -X DELETE "#{url}"|
|
248
|
+
logged(id, curl)
|
249
|
+
end
|
250
|
+
|
251
|
+
def retrieve(type, id, options={})
|
252
|
+
raise ArgumentError, "Please pass a document ID" unless id
|
253
|
+
|
254
|
+
type = Utils.escape(type)
|
255
|
+
url = "#{self.url}/#{type}/#{id}"
|
256
|
+
|
257
|
+
params = {}
|
258
|
+
params[:routing] = options[:routing] if options[:routing]
|
259
|
+
params[:fields] = options[:fields] if options[:fields]
|
260
|
+
params[:preference] = options[:preference] if options[:preference]
|
261
|
+
params_encoded = params.empty? ? '' : "?#{params.to_param}"
|
262
|
+
|
263
|
+
@response = Configuration.client.get "#{url}#{params_encoded}"
|
264
|
+
|
265
|
+
h = MultiJson.decode(@response.body)
|
266
|
+
wrapper = options[:wrapper] || Configuration.wrapper
|
267
|
+
if wrapper == Hash then h
|
268
|
+
else
|
269
|
+
return nil if h['exists'] == false
|
270
|
+
document = h['_source'] || h['fields'] || {}
|
271
|
+
document.update('id' => h['_id'], '_type' => h['_type'], '_index' => h['_index'], '_version' => h['_version'])
|
272
|
+
wrapper.new(document)
|
273
|
+
end
|
274
|
+
|
275
|
+
ensure
|
276
|
+
curl = %Q|curl -X GET "#{url}"|
|
277
|
+
logged(id, curl)
|
278
|
+
end
|
279
|
+
|
280
|
+
def update(type, id, payload={}, options={})
|
281
|
+
raise ArgumentError, "Please pass a document type" unless type
|
282
|
+
raise ArgumentError, "Please pass a document ID" unless id
|
283
|
+
raise ArgumentError, "Please pass a script or partial document in the payload hash" unless payload[:script] || payload[:doc]
|
284
|
+
|
285
|
+
type = Utils.escape(type)
|
286
|
+
url = "#{self.url}/#{type}/#{id}/_update"
|
287
|
+
url += "?#{options.to_param}" unless options.keys.empty?
|
288
|
+
@response = Configuration.client.post url, MultiJson.encode(payload)
|
289
|
+
MultiJson.decode(@response.body)
|
290
|
+
|
291
|
+
ensure
|
292
|
+
curl = %Q|curl -X POST "#{url}" -d '#{MultiJson.encode(payload, :pretty => Configuration.pretty)}'|
|
293
|
+
logged(id, curl)
|
294
|
+
end
|
295
|
+
|
296
|
+
def refresh
|
297
|
+
@response = Configuration.client.post "#{url}/_refresh", ''
|
298
|
+
|
299
|
+
ensure
|
300
|
+
curl = %Q|curl -X POST "#{url}/_refresh"|
|
301
|
+
logged('_refresh', curl)
|
302
|
+
end
|
303
|
+
|
304
|
+
def open(options={})
|
305
|
+
# TODO: Remove the duplication in the execute > rescue > ensure chain
|
306
|
+
@response = Configuration.client.post "#{url}/_open", MultiJson.encode(options)
|
307
|
+
MultiJson.decode(@response.body)['ok']
|
308
|
+
|
309
|
+
ensure
|
310
|
+
curl = %Q|curl -X POST "#{url}/_open"|
|
311
|
+
logged('_open', curl)
|
312
|
+
end
|
313
|
+
|
314
|
+
def close(options={})
|
315
|
+
@response = Configuration.client.post "#{url}/_close", MultiJson.encode(options)
|
316
|
+
MultiJson.decode(@response.body)['ok']
|
317
|
+
|
318
|
+
ensure
|
319
|
+
curl = %Q|curl -X POST "#{url}/_close"|
|
320
|
+
logged('_close', curl)
|
321
|
+
end
|
322
|
+
|
323
|
+
def analyze(text, options={})
|
324
|
+
options = {:pretty => true}.update(options)
|
325
|
+
params = options.to_param
|
326
|
+
@response = Configuration.client.get "#{url}/_analyze?#{params}", text
|
327
|
+
@response.success? ? MultiJson.decode(@response.body) : false
|
328
|
+
|
329
|
+
ensure
|
330
|
+
curl = %Q|curl -X GET "#{url}/_analyze?#{params}" -d '#{text}'|
|
331
|
+
logged('_analyze', curl)
|
332
|
+
end
|
333
|
+
|
334
|
+
def register_percolator_query(name, options={}, &block)
|
335
|
+
options[:query] = Search::Query.new(&block).to_hash if block_given?
|
336
|
+
|
337
|
+
@response = Configuration.client.put "#{Configuration.url}/_percolator/#{@name}/#{name}", MultiJson.encode(options)
|
338
|
+
MultiJson.decode(@response.body)['ok']
|
339
|
+
|
340
|
+
ensure
|
341
|
+
curl = %Q|curl -X PUT "#{Configuration.url}/_percolator/#{@name}/#{name}?pretty" -d '#{MultiJson.encode(options, :pretty => Configuration.pretty)}'|
|
342
|
+
logged('_percolator', curl)
|
343
|
+
end
|
344
|
+
|
345
|
+
def unregister_percolator_query(name)
|
346
|
+
@response = Configuration.client.delete "#{Configuration.url}/_percolator/#{@name}/#{name}"
|
347
|
+
MultiJson.decode(@response.body)['ok']
|
348
|
+
|
349
|
+
ensure
|
350
|
+
curl = %Q|curl -X DELETE "#{Configuration.url}/_percolator/#{@name}"|
|
351
|
+
logged('_percolator', curl)
|
352
|
+
end
|
353
|
+
|
354
|
+
def percolate(*args, &block)
|
355
|
+
document = args.shift
|
356
|
+
type = get_type_from_document(document)
|
357
|
+
|
358
|
+
document = MultiJson.decode convert_document_to_json(document)
|
359
|
+
|
360
|
+
query = Search::Query.new(&block).to_hash if block_given?
|
361
|
+
|
362
|
+
payload = { :doc => document }
|
363
|
+
payload.update( :query => query ) if query
|
364
|
+
|
365
|
+
@response = Configuration.client.get "#{url}/#{type}/_percolate", MultiJson.encode(payload)
|
366
|
+
MultiJson.decode(@response.body)['tire_matches']
|
367
|
+
|
368
|
+
ensure
|
369
|
+
curl = %Q|curl -X GET "#{url}/#{type}/_percolate?pretty" -d '#{MultiJson.encode(payload, :pretty => Configuration.pretty)}'|
|
370
|
+
logged('_percolate', curl)
|
371
|
+
end
|
372
|
+
|
373
|
+
def logged(endpoint='/', curl='')
|
374
|
+
if Configuration.logger
|
375
|
+
error = $!
|
376
|
+
|
377
|
+
Configuration.logger.log_request endpoint, @name, curl
|
378
|
+
|
379
|
+
code = @response ? @response.code : error.class rescue 'N/A'
|
380
|
+
|
381
|
+
if Configuration.logger.level.to_s == 'debug'
|
382
|
+
body = if @response && @response.body && !@response.body.to_s.empty?
|
383
|
+
MultiJson.encode( MultiJson.load(@response.body), :pretty => Configuration.pretty)
|
384
|
+
elsif error && error.message && !error.message.to_s.empty?
|
385
|
+
MultiJson.encode( MultiJson.load(error.message), :pretty => Configuration.pretty) rescue ''
|
386
|
+
else ''
|
387
|
+
end
|
388
|
+
else
|
389
|
+
body = ''
|
390
|
+
end
|
391
|
+
|
392
|
+
Configuration.logger.log_response code, nil, body
|
393
|
+
end
|
394
|
+
end
|
395
|
+
|
396
|
+
def get_type_from_document(document, options={})
|
397
|
+
options = {:escape => true}.merge(options)
|
398
|
+
|
399
|
+
old_verbose, $VERBOSE = $VERBOSE, nil # Silence Object#type deprecation warnings
|
400
|
+
type = case
|
401
|
+
when document.respond_to?(:document_type)
|
402
|
+
document.document_type
|
403
|
+
when document.is_a?(Hash)
|
404
|
+
document[:_type] || document['_type'] || document[:type] || document['type']
|
405
|
+
when document.respond_to?(:_type)
|
406
|
+
document._type
|
407
|
+
when document.respond_to?(:type) && document.type != document.class
|
408
|
+
document.type
|
409
|
+
end
|
410
|
+
$VERBOSE = old_verbose
|
411
|
+
|
412
|
+
type ||= 'document'
|
413
|
+
options[:escape] ? Utils.escape(type) : type
|
414
|
+
end
|
415
|
+
|
416
|
+
def get_id_from_document(document)
|
417
|
+
old_verbose, $VERBOSE = $VERBOSE, nil # Silence Object#id deprecation warnings
|
418
|
+
id = case
|
419
|
+
when document.is_a?(Hash)
|
420
|
+
document[:_id] || document['_id'] || document[:id] || document['id']
|
421
|
+
when document.respond_to?(:id) && document.id != document.object_id
|
422
|
+
document.id.as_json
|
423
|
+
end
|
424
|
+
$VERBOSE = old_verbose
|
425
|
+
id
|
426
|
+
end
|
427
|
+
|
428
|
+
def convert_document_to_json(document)
|
429
|
+
document = case
|
430
|
+
when document.is_a?(String)
|
431
|
+
if ENV['DEBUG']
|
432
|
+
Tire.warn "Passing the document as JSON string has been deprecated, " +
|
433
|
+
"please pass an object which responds to `to_indexed_json` or a plain Hash."
|
434
|
+
end
|
435
|
+
document
|
436
|
+
when document.respond_to?(:to_indexed_json) then document.to_indexed_json
|
437
|
+
else raise ArgumentError, "Please pass a JSON string or object with a 'to_indexed_json' method," +
|
438
|
+
"'#{document.class}' given."
|
439
|
+
end
|
440
|
+
end
|
441
|
+
|
442
|
+
end
|
443
|
+
end
|