es_client 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,17 @@
1
+ module EsClient
2
+ module ActiveRecord
3
+ module Shortcuts
4
+ extend ActiveSupport::Concern
5
+
6
+ included do
7
+ alias_method :es_doc, :es_client_document
8
+ end
9
+
10
+ module ClassMethods
11
+ def es_find(*args)
12
+ es_client.find(*args)
13
+ end
14
+ end
15
+ end
16
+ end
17
+ end
@@ -0,0 +1,71 @@
1
+ module EsClient
2
+ class Client
3
+ RETRY_TIMES = 1
4
+
5
+ def initialize(host, options)
6
+ @host = host
7
+ @options = options
8
+ end
9
+
10
+ def get(path, options={})
11
+ request options.merge(method: :get, path: path)
12
+ end
13
+
14
+ def post(path, options={})
15
+ request options.merge(method: :post, path: path)
16
+ end
17
+
18
+ def put(path, options={})
19
+ request options.merge(method: :put, path: path)
20
+ end
21
+
22
+ def delete(path, options={})
23
+ request options.merge(method: :delete, path: path)
24
+ end
25
+
26
+ def head(path, options={})
27
+ request options.merge(method: :head, path: path)
28
+ end
29
+
30
+ def request(options)
31
+ retry_times = 0
32
+ begin
33
+ raw_response = http.request(options)
34
+ response = ::EsClient::Response.new(raw_response.body, raw_response.status, raw_response.headers)
35
+ EsClient.logger.request(http, response, options) if EsClient.logger.try!(:debug?)
36
+ response
37
+ rescue Excon::Errors::SocketError => e
38
+ if retry_times >= RETRY_TIMES
39
+ exception = ::EsClient::Client::Error.new(e, self)
40
+ EsClient.logger.exception(exception, http, options) if EsClient.logger
41
+ raise exception
42
+ end
43
+ retry_times += 1
44
+ reconnect!
45
+ retry
46
+ end
47
+ end
48
+
49
+ def http
50
+ @http ||= Excon.new(@host, @options)
51
+ end
52
+
53
+ def reconnect!
54
+ @http = nil
55
+ end
56
+
57
+ def log(message, level=:info)
58
+ EsClient.logger.try!(level, message)
59
+ end
60
+
61
+ class Error < StandardError
62
+ attr_reader :transport
63
+
64
+ def initialize(excon_error, transport)
65
+ @transport = transport
66
+ super("#{excon_error.message} (#{excon_error.class})")
67
+ set_backtrace(excon_error.backtrace)
68
+ end
69
+ end
70
+ end
71
+ end
@@ -0,0 +1,93 @@
1
+ module EsClient
2
+ class Index
3
+ attr_reader :name, :options
4
+
5
+ def initialize(name, options={})
6
+ @name = build_name(name)
7
+ @options = options
8
+ end
9
+
10
+ def build_name(name)
11
+ return name unless EsClient.index_prefix
12
+ "#{EsClient.index_prefix}_#{name}"
13
+ end
14
+
15
+ def exists?
16
+ EsClient.client.head("/#{name}").success?
17
+ end
18
+
19
+ def recreate
20
+ delete
21
+ create
22
+ end
23
+
24
+ def create
25
+ request_options = @options.present? ? {body: @options.to_json} : {}
26
+ EsClient.client.post("/#{name}", request_options)
27
+ end
28
+
29
+ def delete
30
+ EsClient.client.delete("/#{name}")
31
+ end
32
+
33
+ def refresh
34
+ EsClient.client.post("/#{name}/_refresh")
35
+ end
36
+
37
+ def search(query, options={})
38
+ http_options = options.slice(:query, :headers)
39
+ http_options[:body] = query.to_json
40
+ EsClient.client.get("/#{name}/#{options[:type]}/_search", http_options)
41
+ end
42
+
43
+ def get_settings
44
+ EsClient.client.get("/#{name}/_settings").decoded[name]['settings']
45
+ end
46
+
47
+ def put_settings(settings)
48
+ EsClient.client.put("/#{name}/_settings", body: settings.to_json)
49
+ end
50
+
51
+ def get_mapping
52
+ EsClient.client.get("/#{name}/_mapping").decoded[name]['mappings']
53
+ end
54
+
55
+ def put_mapping(type, mapping)
56
+ json = {type => mapping}.to_json
57
+ EsClient.client.put("/#{name}/_mapping/#{type}", body: json)
58
+ end
59
+
60
+ def save_document(type, id, document)
61
+ EsClient.client.post("/#{name}/#{type}/#{id}", body: document.to_json)
62
+ end
63
+
64
+ def update_document(type, id, document)
65
+ EsClient.client.post("/#{name}/#{type}/#{id}/_update", body: {doc: document}.to_json)
66
+ end
67
+
68
+ def destroy_document(type, id)
69
+ EsClient.client.delete("/#{name}/#{type}/#{id}")
70
+ end
71
+
72
+ def find(type, id)
73
+ EsClient.client.get("/#{name}/#{type}/#{id}").decoded['_source']
74
+ end
75
+
76
+ def bulk(action, type, documents)
77
+ payload = []
78
+ documents.each do |document|
79
+ payload << {action => {_index: name, _type: type, _id: document[:id]}}
80
+ case action
81
+ when :index
82
+ payload << document
83
+ when :update
84
+ document_for_update = {doc: document}
85
+ document_for_update.update(document[:bulk_options]) if document[:bulk_options]
86
+ payload << document_for_update
87
+ end
88
+ end
89
+ serialized_payload = "\n" + payload.map(&:to_json).join("\n") + "\n"
90
+ EsClient.client.post("/#{name}/#{type}/_bulk", body: serialized_payload)
91
+ end
92
+ end
93
+ end
@@ -0,0 +1,57 @@
1
+ module EsClient
2
+ class Logger < ::Logger
3
+ def initialize(path, options)
4
+ super(path)
5
+ @options = options
6
+ end
7
+
8
+ def request(http, response, options)
9
+ return unless debug?
10
+ took = response.try!(:decoded).try!(:[], 'took') ? response.decoded['took'] : 'N/A'
11
+ message = "[#{response.code}](#{took} msec) #{to_curl(http, options)}"
12
+ message << "\n#{JSON.pretty_generate(response.decoded)}" if @options[:log_response] && response.try!(:decoded)
13
+ debug message
14
+ end
15
+
16
+ def exception(e, http=nil, options=nil)
17
+ backtrace = e.backtrace.map { |l| "#{' ' * 2}#{l}" }.join("\n")
18
+ curl = "\n #{to_curl(http, options)}" if options && http
19
+ error "#{e.class} #{e.message} #{curl}\n#{backtrace}\n\n"
20
+ end
21
+
22
+ private
23
+
24
+ def to_curl(http, options)
25
+ res = 'curl -i -X '
26
+ res << options[:method].to_s.upcase
27
+
28
+ res << " '#{http.data[:scheme]}://#{http.data[:host]}"
29
+ res << ":#{http.data[:port]}" if http.data[:port]
30
+ res << options[:path]
31
+ if options[:query].present?
32
+ res << '?'
33
+ res << options[:query].is_a?(String) ? options[:query] : options[:query].to_query
34
+ elsif @options[:pretty]
35
+ res << '?'
36
+ end
37
+ res << '&pretty' if @options[:pretty]
38
+ res << "'"
39
+
40
+ if options[:body]
41
+ if options[:path].include?('/_bulk')
42
+ binary_data = @options[:log_binary] ? options[:body] : '... data omitted ...'
43
+ res << " --data-binary '#{binary_data}'"
44
+ else
45
+ res << " -d '#{pretty_json(options[:body])}'"
46
+ end
47
+ end
48
+ res
49
+ end
50
+
51
+ def pretty_json(string)
52
+ return if string.blank?
53
+ return string unless @options[:pretty]
54
+ JSON.pretty_generate(JSON.parse(string)).gsub("'", '\u0027')
55
+ end
56
+ end
57
+ end
@@ -0,0 +1,24 @@
1
+ module EsClient
2
+ class Response
3
+ attr_reader :body, :code, :headers
4
+
5
+ def initialize(body, code, headers={})
6
+ @body = body
7
+ @code = code.to_i
8
+ @headers = headers
9
+ end
10
+
11
+ def success?
12
+ code > 0 && code < 400
13
+ end
14
+
15
+ def failure?
16
+ !success?
17
+ end
18
+
19
+ def decoded
20
+ return if @body.blank?
21
+ @decoded ||= JSON.parse(@body)
22
+ end
23
+ end
24
+ end
@@ -0,0 +1,3 @@
1
+ module EsClient
2
+ VERSION = '0.1.0'
3
+ end
@@ -0,0 +1,106 @@
1
+ require 'spec_helper'
2
+
3
+ describe EsClient::ActiveRecord::Adapter do
4
+ describe 'index name' do
5
+ it 'determine index name from model' do
6
+ expect(RspecUser.es_client.index_name).to eq 'rspec_users'
7
+ end
8
+
9
+ it 'allow to define custom index name' do
10
+ RspecUser.es_client.index_name('custom_index_name')
11
+ expect(RspecUser.es_client.index_name).to eq 'custom_index_name'
12
+ RspecUser.es_client.instance_variable_set(:@index_name, nil)
13
+ end
14
+ end
15
+
16
+ describe 'document type' do
17
+ it 'determine document type from model' do
18
+ expect(RspecUser.es_client.document_type).to eq 'rspec_user'
19
+ end
20
+
21
+ it 'allow to define custom document type' do
22
+ RspecUser.es_client.document_type('custom_document_type')
23
+ expect(RspecUser.es_client.document_type).to eq 'custom_document_type'
24
+ RspecUser.es_client.instance_variable_set(:@document_type, nil)
25
+ end
26
+ end
27
+
28
+ it 'save document' do
29
+ expect(RspecUser.es_client.index).to receive(:save_document).with('rspec_user', 1, {id: 1, name: 'bob'})
30
+ RspecUser.es_client.save_document(RspecUser.new(id: 1, name: 'bob'))
31
+ end
32
+
33
+ describe 'update document' do
34
+ it 'update document' do
35
+ expect(RspecUser.es_client.index).to receive(:update_document).with('rspec_user', 1, {name: 'arnold'})
36
+ record = RspecUser.new(id: 1, name: 'bob')
37
+ allow(record).to receive(:changes).and_return({name: %w(bob arnold)})
38
+ RspecUser.es_client.update_document(record)
39
+ end
40
+ end
41
+
42
+ it 'destroy document' do
43
+ expect(RspecUser.es_client.index).to receive(:destroy_document).with('rspec_user', 1)
44
+ RspecUser.es_client.destroy_document(1)
45
+ end
46
+
47
+ describe 'find' do
48
+ it 'find document' do
49
+ expect(RspecUser.es_client.index).to receive(:find).with('rspec_user', 1)
50
+ RspecUser.es_client.find(1)
51
+ end
52
+
53
+ it 'find multiple documents' do
54
+ expect(RspecUser.es_client.index).to receive(:search).with({query: {ids: {values: [1], type: 'rspec_user'}}, size: 1}, type: 'rspec_user')
55
+ RspecUser.es_client.find([1])
56
+ end
57
+ end
58
+
59
+ describe 'import' do
60
+ it 'import batch of records' do
61
+ expect(RspecUser.es_client.index).to receive(:bulk).with(:index, 'rspec_user', [{id: 1}])
62
+ RspecUser.es_client.import([RspecUser.new(id: 1)])
63
+ end
64
+ end
65
+
66
+ describe 'mapping' do
67
+ it 'fetch mapping' do
68
+ expect(RspecUser.es_client.index).to receive(:get_mapping)
69
+ RspecUser.es_client.mapping
70
+ end
71
+
72
+ it 'set mapping' do
73
+ RspecUser.es_client.index.options[:mappings] = {}
74
+ RspecUser.es_client.mapping(test: {properties: {notes: {type: 'string'}}})
75
+ expect(RspecUser.es_client.index.options[:mappings]).to include(test: {properties: {notes: {type: 'string'}}})
76
+ end
77
+
78
+ it 'set append mapping' do
79
+ RspecUser.es_client.index.options[:mappings] = {}
80
+ RspecUser.es_client.mapping(test: {properties: {prop1: {type: 'string'}}})
81
+ RspecUser.es_client.mapping(test: {properties: {prop2: {type: 'string'}}})
82
+ expect(RspecUser.es_client.index.options[:mappings][:test][:properties]).to include(prop1: {type: 'string'})
83
+ expect(RspecUser.es_client.index.options[:mappings][:test][:properties]).to include(prop2: {type: 'string'})
84
+ end
85
+ end
86
+
87
+ describe 'settings' do
88
+ it 'fetch settings' do
89
+ expect(RspecUser.es_client.index).to receive(:get_settings)
90
+ RspecUser.es_client.settings
91
+ end
92
+
93
+ it 'set settings' do
94
+ RspecUser.es_client.index.options[:settings] = {}
95
+ RspecUser.es_client.settings(refresh_interval: '3s')
96
+ expect(RspecUser.es_client.index.options[:settings]).to include(refresh_interval: '3s')
97
+ end
98
+ end
99
+
100
+ describe 'search' do
101
+ it 'perform search query' do
102
+ expect(RspecUser.es_client.index).to receive(:search).with({query: {query_string: {query: 'test'}}}, type: 'rspec_user')
103
+ RspecUser.es_client.search(query: {query_string: {query: 'test'}})
104
+ end
105
+ end
106
+ end
@@ -0,0 +1,68 @@
1
+ require 'spec_helper'
2
+
3
+ describe EsClient::ActiveRecord::Glue do
4
+ describe 'callbacks' do
5
+ it 'save document after save' do
6
+ expect(RspecUser.es_client).to receive(:save_document).with(instance_of(RspecUser))
7
+ RspecUser.new(id: 1, name: 'bob').save
8
+ end
9
+
10
+ it 'destroy document after destroy' do
11
+ expect(RspecUser.es_client).to receive(:destroy_document).with(1)
12
+ RspecUser.new(id: 1, name: 'bob').destroy
13
+ end
14
+
15
+ it 'allow to disable callbacks' do
16
+ allow(EsClient).to receive(:callbacks_enabled).and_return(false)
17
+ expect(RspecUser.es_client).not_to receive(:save_document)
18
+ RspecUser.new(id: 1, name: 'bob').save
19
+ end
20
+ end
21
+
22
+ describe 'update' do
23
+ it 'update es document' do
24
+ expect(RspecUser.es_client).to receive(:update_document)
25
+ RspecUser.new(id: 1, name: 'bob').es_client_update
26
+ end
27
+
28
+ it 'do not update new record document' do
29
+ expect(RspecUser.es_client).not_to receive(:update_document)
30
+ record = RspecUser.new(id: 1, name: 'bob')
31
+ allow(record).to receive(:new_record?).and_return(true)
32
+ record.es_client_update
33
+ end
34
+ end
35
+
36
+ describe 'record es document' do
37
+ it 'return es document' do
38
+ expect(RspecUser.es_client).to receive(:find).with(1)
39
+ RspecUser.new(id: 1, name: 'bob').es_client_document
40
+ end
41
+
42
+ it 'fetch es document once' do
43
+ expect(RspecUser.es_client).to receive(:find).with(1).once
44
+ record = RspecUser.new(id: 1, name: 'bob')
45
+ record.es_client_document
46
+ record.es_client_document
47
+ end
48
+
49
+ it 'force fetch es document' do
50
+ expect(RspecUser.es_client).to receive(:find).with(1).twice
51
+ record = RspecUser.new(id: 1, name: 'bob')
52
+ record.es_client_document
53
+ record.es_client_document(true)
54
+ end
55
+ end
56
+
57
+ describe 'reindex' do
58
+ it 'reindex current scope' do
59
+ expect(RspecUser.es_client).to receive(:import).twice.with(instance_of(Array))
60
+ RspecUser.es_client_reindex
61
+ end
62
+
63
+ it 'reindex current scope with progress' do
64
+ expect(RspecUser.es_client).to receive(:import).twice.with(instance_of(Array))
65
+ RspecUser.es_client_reindex_with_progress(batch_size: 1)
66
+ end
67
+ end
68
+ end