es_client 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.gitignore +18 -0
- data/.rspec +2 -0
- data/.travis.yml +3 -0
- data/Gemfile +8 -0
- data/Guardfile +48 -0
- data/LICENSE.txt +22 -0
- data/README.md +99 -0
- data/ROADMAP.md +9 -0
- data/Rakefile +7 -0
- data/es_client.gemspec +29 -0
- data/lib/es_client.rb +45 -0
- data/lib/es_client/active_record/adapter.rb +81 -0
- data/lib/es_client/active_record/glue.rb +72 -0
- data/lib/es_client/active_record/shortcuts.rb +17 -0
- data/lib/es_client/client.rb +71 -0
- data/lib/es_client/index.rb +93 -0
- data/lib/es_client/logger.rb +57 -0
- data/lib/es_client/response.rb +24 -0
- data/lib/es_client/version.rb +3 -0
- data/spec/es_client/active_record/adapter_spec.rb +106 -0
- data/spec/es_client/active_record/glue_spec.rb +68 -0
- data/spec/es_client/active_record/shortcuts_spec.rb +13 -0
- data/spec/es_client/index_spec.rb +40 -0
- data/spec/es_client/logger_spec.rb +18 -0
- data/spec/es_client/responce_spec.rb +25 -0
- data/spec/es_client/transport_spec.rb +51 -0
- data/spec/es_client_spec.rb +40 -0
- data/spec/integration/es_client/index_spec.rb +156 -0
- data/spec/spec_helper.rb +113 -0
- metadata +182 -0
@@ -0,0 +1,17 @@
|
|
1
|
+
module EsClient
|
2
|
+
module ActiveRecord
|
3
|
+
module Shortcuts
|
4
|
+
extend ActiveSupport::Concern
|
5
|
+
|
6
|
+
included do
|
7
|
+
alias_method :es_doc, :es_client_document
|
8
|
+
end
|
9
|
+
|
10
|
+
module ClassMethods
|
11
|
+
def es_find(*args)
|
12
|
+
es_client.find(*args)
|
13
|
+
end
|
14
|
+
end
|
15
|
+
end
|
16
|
+
end
|
17
|
+
end
|
@@ -0,0 +1,71 @@
|
|
1
|
+
module EsClient
|
2
|
+
class Client
|
3
|
+
RETRY_TIMES = 1
|
4
|
+
|
5
|
+
def initialize(host, options)
|
6
|
+
@host = host
|
7
|
+
@options = options
|
8
|
+
end
|
9
|
+
|
10
|
+
def get(path, options={})
|
11
|
+
request options.merge(method: :get, path: path)
|
12
|
+
end
|
13
|
+
|
14
|
+
def post(path, options={})
|
15
|
+
request options.merge(method: :post, path: path)
|
16
|
+
end
|
17
|
+
|
18
|
+
def put(path, options={})
|
19
|
+
request options.merge(method: :put, path: path)
|
20
|
+
end
|
21
|
+
|
22
|
+
def delete(path, options={})
|
23
|
+
request options.merge(method: :delete, path: path)
|
24
|
+
end
|
25
|
+
|
26
|
+
def head(path, options={})
|
27
|
+
request options.merge(method: :head, path: path)
|
28
|
+
end
|
29
|
+
|
30
|
+
def request(options)
|
31
|
+
retry_times = 0
|
32
|
+
begin
|
33
|
+
raw_response = http.request(options)
|
34
|
+
response = ::EsClient::Response.new(raw_response.body, raw_response.status, raw_response.headers)
|
35
|
+
EsClient.logger.request(http, response, options) if EsClient.logger.try!(:debug?)
|
36
|
+
response
|
37
|
+
rescue Excon::Errors::SocketError => e
|
38
|
+
if retry_times >= RETRY_TIMES
|
39
|
+
exception = ::EsClient::Client::Error.new(e, self)
|
40
|
+
EsClient.logger.exception(exception, http, options) if EsClient.logger
|
41
|
+
raise exception
|
42
|
+
end
|
43
|
+
retry_times += 1
|
44
|
+
reconnect!
|
45
|
+
retry
|
46
|
+
end
|
47
|
+
end
|
48
|
+
|
49
|
+
def http
|
50
|
+
@http ||= Excon.new(@host, @options)
|
51
|
+
end
|
52
|
+
|
53
|
+
def reconnect!
|
54
|
+
@http = nil
|
55
|
+
end
|
56
|
+
|
57
|
+
def log(message, level=:info)
|
58
|
+
EsClient.logger.try!(level, message)
|
59
|
+
end
|
60
|
+
|
61
|
+
class Error < StandardError
|
62
|
+
attr_reader :transport
|
63
|
+
|
64
|
+
def initialize(excon_error, transport)
|
65
|
+
@transport = transport
|
66
|
+
super("#{excon_error.message} (#{excon_error.class})")
|
67
|
+
set_backtrace(excon_error.backtrace)
|
68
|
+
end
|
69
|
+
end
|
70
|
+
end
|
71
|
+
end
|
@@ -0,0 +1,93 @@
|
|
1
|
+
module EsClient
|
2
|
+
class Index
|
3
|
+
attr_reader :name, :options
|
4
|
+
|
5
|
+
def initialize(name, options={})
|
6
|
+
@name = build_name(name)
|
7
|
+
@options = options
|
8
|
+
end
|
9
|
+
|
10
|
+
def build_name(name)
|
11
|
+
return name unless EsClient.index_prefix
|
12
|
+
"#{EsClient.index_prefix}_#{name}"
|
13
|
+
end
|
14
|
+
|
15
|
+
def exists?
|
16
|
+
EsClient.client.head("/#{name}").success?
|
17
|
+
end
|
18
|
+
|
19
|
+
def recreate
|
20
|
+
delete
|
21
|
+
create
|
22
|
+
end
|
23
|
+
|
24
|
+
def create
|
25
|
+
request_options = @options.present? ? {body: @options.to_json} : {}
|
26
|
+
EsClient.client.post("/#{name}", request_options)
|
27
|
+
end
|
28
|
+
|
29
|
+
def delete
|
30
|
+
EsClient.client.delete("/#{name}")
|
31
|
+
end
|
32
|
+
|
33
|
+
def refresh
|
34
|
+
EsClient.client.post("/#{name}/_refresh")
|
35
|
+
end
|
36
|
+
|
37
|
+
def search(query, options={})
|
38
|
+
http_options = options.slice(:query, :headers)
|
39
|
+
http_options[:body] = query.to_json
|
40
|
+
EsClient.client.get("/#{name}/#{options[:type]}/_search", http_options)
|
41
|
+
end
|
42
|
+
|
43
|
+
def get_settings
|
44
|
+
EsClient.client.get("/#{name}/_settings").decoded[name]['settings']
|
45
|
+
end
|
46
|
+
|
47
|
+
def put_settings(settings)
|
48
|
+
EsClient.client.put("/#{name}/_settings", body: settings.to_json)
|
49
|
+
end
|
50
|
+
|
51
|
+
def get_mapping
|
52
|
+
EsClient.client.get("/#{name}/_mapping").decoded[name]['mappings']
|
53
|
+
end
|
54
|
+
|
55
|
+
def put_mapping(type, mapping)
|
56
|
+
json = {type => mapping}.to_json
|
57
|
+
EsClient.client.put("/#{name}/_mapping/#{type}", body: json)
|
58
|
+
end
|
59
|
+
|
60
|
+
def save_document(type, id, document)
|
61
|
+
EsClient.client.post("/#{name}/#{type}/#{id}", body: document.to_json)
|
62
|
+
end
|
63
|
+
|
64
|
+
def update_document(type, id, document)
|
65
|
+
EsClient.client.post("/#{name}/#{type}/#{id}/_update", body: {doc: document}.to_json)
|
66
|
+
end
|
67
|
+
|
68
|
+
def destroy_document(type, id)
|
69
|
+
EsClient.client.delete("/#{name}/#{type}/#{id}")
|
70
|
+
end
|
71
|
+
|
72
|
+
def find(type, id)
|
73
|
+
EsClient.client.get("/#{name}/#{type}/#{id}").decoded['_source']
|
74
|
+
end
|
75
|
+
|
76
|
+
def bulk(action, type, documents)
|
77
|
+
payload = []
|
78
|
+
documents.each do |document|
|
79
|
+
payload << {action => {_index: name, _type: type, _id: document[:id]}}
|
80
|
+
case action
|
81
|
+
when :index
|
82
|
+
payload << document
|
83
|
+
when :update
|
84
|
+
document_for_update = {doc: document}
|
85
|
+
document_for_update.update(document[:bulk_options]) if document[:bulk_options]
|
86
|
+
payload << document_for_update
|
87
|
+
end
|
88
|
+
end
|
89
|
+
serialized_payload = "\n" + payload.map(&:to_json).join("\n") + "\n"
|
90
|
+
EsClient.client.post("/#{name}/#{type}/_bulk", body: serialized_payload)
|
91
|
+
end
|
92
|
+
end
|
93
|
+
end
|
@@ -0,0 +1,57 @@
|
|
1
|
+
module EsClient
|
2
|
+
class Logger < ::Logger
|
3
|
+
def initialize(path, options)
|
4
|
+
super(path)
|
5
|
+
@options = options
|
6
|
+
end
|
7
|
+
|
8
|
+
def request(http, response, options)
|
9
|
+
return unless debug?
|
10
|
+
took = response.try!(:decoded).try!(:[], 'took') ? response.decoded['took'] : 'N/A'
|
11
|
+
message = "[#{response.code}](#{took} msec) #{to_curl(http, options)}"
|
12
|
+
message << "\n#{JSON.pretty_generate(response.decoded)}" if @options[:log_response] && response.try!(:decoded)
|
13
|
+
debug message
|
14
|
+
end
|
15
|
+
|
16
|
+
def exception(e, http=nil, options=nil)
|
17
|
+
backtrace = e.backtrace.map { |l| "#{' ' * 2}#{l}" }.join("\n")
|
18
|
+
curl = "\n #{to_curl(http, options)}" if options && http
|
19
|
+
error "#{e.class} #{e.message} #{curl}\n#{backtrace}\n\n"
|
20
|
+
end
|
21
|
+
|
22
|
+
private
|
23
|
+
|
24
|
+
def to_curl(http, options)
|
25
|
+
res = 'curl -i -X '
|
26
|
+
res << options[:method].to_s.upcase
|
27
|
+
|
28
|
+
res << " '#{http.data[:scheme]}://#{http.data[:host]}"
|
29
|
+
res << ":#{http.data[:port]}" if http.data[:port]
|
30
|
+
res << options[:path]
|
31
|
+
if options[:query].present?
|
32
|
+
res << '?'
|
33
|
+
res << options[:query].is_a?(String) ? options[:query] : options[:query].to_query
|
34
|
+
elsif @options[:pretty]
|
35
|
+
res << '?'
|
36
|
+
end
|
37
|
+
res << '&pretty' if @options[:pretty]
|
38
|
+
res << "'"
|
39
|
+
|
40
|
+
if options[:body]
|
41
|
+
if options[:path].include?('/_bulk')
|
42
|
+
binary_data = @options[:log_binary] ? options[:body] : '... data omitted ...'
|
43
|
+
res << " --data-binary '#{binary_data}'"
|
44
|
+
else
|
45
|
+
res << " -d '#{pretty_json(options[:body])}'"
|
46
|
+
end
|
47
|
+
end
|
48
|
+
res
|
49
|
+
end
|
50
|
+
|
51
|
+
def pretty_json(string)
|
52
|
+
return if string.blank?
|
53
|
+
return string unless @options[:pretty]
|
54
|
+
JSON.pretty_generate(JSON.parse(string)).gsub("'", '\u0027')
|
55
|
+
end
|
56
|
+
end
|
57
|
+
end
|
@@ -0,0 +1,24 @@
|
|
1
|
+
module EsClient
|
2
|
+
class Response
|
3
|
+
attr_reader :body, :code, :headers
|
4
|
+
|
5
|
+
def initialize(body, code, headers={})
|
6
|
+
@body = body
|
7
|
+
@code = code.to_i
|
8
|
+
@headers = headers
|
9
|
+
end
|
10
|
+
|
11
|
+
def success?
|
12
|
+
code > 0 && code < 400
|
13
|
+
end
|
14
|
+
|
15
|
+
def failure?
|
16
|
+
!success?
|
17
|
+
end
|
18
|
+
|
19
|
+
def decoded
|
20
|
+
return if @body.blank?
|
21
|
+
@decoded ||= JSON.parse(@body)
|
22
|
+
end
|
23
|
+
end
|
24
|
+
end
|
@@ -0,0 +1,106 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
|
3
|
+
describe EsClient::ActiveRecord::Adapter do
|
4
|
+
describe 'index name' do
|
5
|
+
it 'determine index name from model' do
|
6
|
+
expect(RspecUser.es_client.index_name).to eq 'rspec_users'
|
7
|
+
end
|
8
|
+
|
9
|
+
it 'allow to define custom index name' do
|
10
|
+
RspecUser.es_client.index_name('custom_index_name')
|
11
|
+
expect(RspecUser.es_client.index_name).to eq 'custom_index_name'
|
12
|
+
RspecUser.es_client.instance_variable_set(:@index_name, nil)
|
13
|
+
end
|
14
|
+
end
|
15
|
+
|
16
|
+
describe 'document type' do
|
17
|
+
it 'determine document type from model' do
|
18
|
+
expect(RspecUser.es_client.document_type).to eq 'rspec_user'
|
19
|
+
end
|
20
|
+
|
21
|
+
it 'allow to define custom document type' do
|
22
|
+
RspecUser.es_client.document_type('custom_document_type')
|
23
|
+
expect(RspecUser.es_client.document_type).to eq 'custom_document_type'
|
24
|
+
RspecUser.es_client.instance_variable_set(:@document_type, nil)
|
25
|
+
end
|
26
|
+
end
|
27
|
+
|
28
|
+
it 'save document' do
|
29
|
+
expect(RspecUser.es_client.index).to receive(:save_document).with('rspec_user', 1, {id: 1, name: 'bob'})
|
30
|
+
RspecUser.es_client.save_document(RspecUser.new(id: 1, name: 'bob'))
|
31
|
+
end
|
32
|
+
|
33
|
+
describe 'update document' do
|
34
|
+
it 'update document' do
|
35
|
+
expect(RspecUser.es_client.index).to receive(:update_document).with('rspec_user', 1, {name: 'arnold'})
|
36
|
+
record = RspecUser.new(id: 1, name: 'bob')
|
37
|
+
allow(record).to receive(:changes).and_return({name: %w(bob arnold)})
|
38
|
+
RspecUser.es_client.update_document(record)
|
39
|
+
end
|
40
|
+
end
|
41
|
+
|
42
|
+
it 'destroy document' do
|
43
|
+
expect(RspecUser.es_client.index).to receive(:destroy_document).with('rspec_user', 1)
|
44
|
+
RspecUser.es_client.destroy_document(1)
|
45
|
+
end
|
46
|
+
|
47
|
+
describe 'find' do
|
48
|
+
it 'find document' do
|
49
|
+
expect(RspecUser.es_client.index).to receive(:find).with('rspec_user', 1)
|
50
|
+
RspecUser.es_client.find(1)
|
51
|
+
end
|
52
|
+
|
53
|
+
it 'find multiple documents' do
|
54
|
+
expect(RspecUser.es_client.index).to receive(:search).with({query: {ids: {values: [1], type: 'rspec_user'}}, size: 1}, type: 'rspec_user')
|
55
|
+
RspecUser.es_client.find([1])
|
56
|
+
end
|
57
|
+
end
|
58
|
+
|
59
|
+
describe 'import' do
|
60
|
+
it 'import batch of records' do
|
61
|
+
expect(RspecUser.es_client.index).to receive(:bulk).with(:index, 'rspec_user', [{id: 1}])
|
62
|
+
RspecUser.es_client.import([RspecUser.new(id: 1)])
|
63
|
+
end
|
64
|
+
end
|
65
|
+
|
66
|
+
describe 'mapping' do
|
67
|
+
it 'fetch mapping' do
|
68
|
+
expect(RspecUser.es_client.index).to receive(:get_mapping)
|
69
|
+
RspecUser.es_client.mapping
|
70
|
+
end
|
71
|
+
|
72
|
+
it 'set mapping' do
|
73
|
+
RspecUser.es_client.index.options[:mappings] = {}
|
74
|
+
RspecUser.es_client.mapping(test: {properties: {notes: {type: 'string'}}})
|
75
|
+
expect(RspecUser.es_client.index.options[:mappings]).to include(test: {properties: {notes: {type: 'string'}}})
|
76
|
+
end
|
77
|
+
|
78
|
+
it 'set append mapping' do
|
79
|
+
RspecUser.es_client.index.options[:mappings] = {}
|
80
|
+
RspecUser.es_client.mapping(test: {properties: {prop1: {type: 'string'}}})
|
81
|
+
RspecUser.es_client.mapping(test: {properties: {prop2: {type: 'string'}}})
|
82
|
+
expect(RspecUser.es_client.index.options[:mappings][:test][:properties]).to include(prop1: {type: 'string'})
|
83
|
+
expect(RspecUser.es_client.index.options[:mappings][:test][:properties]).to include(prop2: {type: 'string'})
|
84
|
+
end
|
85
|
+
end
|
86
|
+
|
87
|
+
describe 'settings' do
|
88
|
+
it 'fetch settings' do
|
89
|
+
expect(RspecUser.es_client.index).to receive(:get_settings)
|
90
|
+
RspecUser.es_client.settings
|
91
|
+
end
|
92
|
+
|
93
|
+
it 'set settings' do
|
94
|
+
RspecUser.es_client.index.options[:settings] = {}
|
95
|
+
RspecUser.es_client.settings(refresh_interval: '3s')
|
96
|
+
expect(RspecUser.es_client.index.options[:settings]).to include(refresh_interval: '3s')
|
97
|
+
end
|
98
|
+
end
|
99
|
+
|
100
|
+
describe 'search' do
|
101
|
+
it 'perform search query' do
|
102
|
+
expect(RspecUser.es_client.index).to receive(:search).with({query: {query_string: {query: 'test'}}}, type: 'rspec_user')
|
103
|
+
RspecUser.es_client.search(query: {query_string: {query: 'test'}})
|
104
|
+
end
|
105
|
+
end
|
106
|
+
end
|
@@ -0,0 +1,68 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
|
3
|
+
describe EsClient::ActiveRecord::Glue do
|
4
|
+
describe 'callbacks' do
|
5
|
+
it 'save document after save' do
|
6
|
+
expect(RspecUser.es_client).to receive(:save_document).with(instance_of(RspecUser))
|
7
|
+
RspecUser.new(id: 1, name: 'bob').save
|
8
|
+
end
|
9
|
+
|
10
|
+
it 'destroy document after destroy' do
|
11
|
+
expect(RspecUser.es_client).to receive(:destroy_document).with(1)
|
12
|
+
RspecUser.new(id: 1, name: 'bob').destroy
|
13
|
+
end
|
14
|
+
|
15
|
+
it 'allow to disable callbacks' do
|
16
|
+
allow(EsClient).to receive(:callbacks_enabled).and_return(false)
|
17
|
+
expect(RspecUser.es_client).not_to receive(:save_document)
|
18
|
+
RspecUser.new(id: 1, name: 'bob').save
|
19
|
+
end
|
20
|
+
end
|
21
|
+
|
22
|
+
describe 'update' do
|
23
|
+
it 'update es document' do
|
24
|
+
expect(RspecUser.es_client).to receive(:update_document)
|
25
|
+
RspecUser.new(id: 1, name: 'bob').es_client_update
|
26
|
+
end
|
27
|
+
|
28
|
+
it 'do not update new record document' do
|
29
|
+
expect(RspecUser.es_client).not_to receive(:update_document)
|
30
|
+
record = RspecUser.new(id: 1, name: 'bob')
|
31
|
+
allow(record).to receive(:new_record?).and_return(true)
|
32
|
+
record.es_client_update
|
33
|
+
end
|
34
|
+
end
|
35
|
+
|
36
|
+
describe 'record es document' do
|
37
|
+
it 'return es document' do
|
38
|
+
expect(RspecUser.es_client).to receive(:find).with(1)
|
39
|
+
RspecUser.new(id: 1, name: 'bob').es_client_document
|
40
|
+
end
|
41
|
+
|
42
|
+
it 'fetch es document once' do
|
43
|
+
expect(RspecUser.es_client).to receive(:find).with(1).once
|
44
|
+
record = RspecUser.new(id: 1, name: 'bob')
|
45
|
+
record.es_client_document
|
46
|
+
record.es_client_document
|
47
|
+
end
|
48
|
+
|
49
|
+
it 'force fetch es document' do
|
50
|
+
expect(RspecUser.es_client).to receive(:find).with(1).twice
|
51
|
+
record = RspecUser.new(id: 1, name: 'bob')
|
52
|
+
record.es_client_document
|
53
|
+
record.es_client_document(true)
|
54
|
+
end
|
55
|
+
end
|
56
|
+
|
57
|
+
describe 'reindex' do
|
58
|
+
it 'reindex current scope' do
|
59
|
+
expect(RspecUser.es_client).to receive(:import).twice.with(instance_of(Array))
|
60
|
+
RspecUser.es_client_reindex
|
61
|
+
end
|
62
|
+
|
63
|
+
it 'reindex current scope with progress' do
|
64
|
+
expect(RspecUser.es_client).to receive(:import).twice.with(instance_of(Array))
|
65
|
+
RspecUser.es_client_reindex_with_progress(batch_size: 1)
|
66
|
+
end
|
67
|
+
end
|
68
|
+
end
|