es_client 0.1.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/.gitignore +18 -0
- data/.rspec +2 -0
- data/.travis.yml +3 -0
- data/Gemfile +8 -0
- data/Guardfile +48 -0
- data/LICENSE.txt +22 -0
- data/README.md +99 -0
- data/ROADMAP.md +9 -0
- data/Rakefile +7 -0
- data/es_client.gemspec +29 -0
- data/lib/es_client.rb +45 -0
- data/lib/es_client/active_record/adapter.rb +81 -0
- data/lib/es_client/active_record/glue.rb +72 -0
- data/lib/es_client/active_record/shortcuts.rb +17 -0
- data/lib/es_client/client.rb +71 -0
- data/lib/es_client/index.rb +93 -0
- data/lib/es_client/logger.rb +57 -0
- data/lib/es_client/response.rb +24 -0
- data/lib/es_client/version.rb +3 -0
- data/spec/es_client/active_record/adapter_spec.rb +106 -0
- data/spec/es_client/active_record/glue_spec.rb +68 -0
- data/spec/es_client/active_record/shortcuts_spec.rb +13 -0
- data/spec/es_client/index_spec.rb +40 -0
- data/spec/es_client/logger_spec.rb +18 -0
- data/spec/es_client/responce_spec.rb +25 -0
- data/spec/es_client/transport_spec.rb +51 -0
- data/spec/es_client_spec.rb +40 -0
- data/spec/integration/es_client/index_spec.rb +156 -0
- data/spec/spec_helper.rb +113 -0
- metadata +182 -0
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA1:
|
3
|
+
metadata.gz: 843f33d86423abe66668f3a433877678d0e82435
|
4
|
+
data.tar.gz: 785f403f7aae35ac33896e67b82b03e809d36c05
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: 647d9d56a31b29d570fc2e0cf175a2a717aa0e055acaf2ecb6ad84414d5b16761eac264a4de07f33d86e5ff1248dc30112f95552964dbdf499cf314615b95cfb
|
7
|
+
data.tar.gz: 6943f76a4645317c49c196ab678a1ee108089516f9a73cc3850b98ad161d261387479e5a37e131357752cb55e747c0645fda60aa842526c6d3494a85195479d5
|
data/.gitignore
ADDED
data/.rspec
ADDED
data/.travis.yml
ADDED
data/Gemfile
ADDED
data/Guardfile
ADDED
@@ -0,0 +1,48 @@
|
|
1
|
+
# A sample Guardfile
|
2
|
+
# More info at https://github.com/guard/guard#readme
|
3
|
+
|
4
|
+
## Uncomment and set this to only include directories you want to watch
|
5
|
+
# directories %w(app lib config test spec features)
|
6
|
+
|
7
|
+
## Uncomment to clear the screen before every task
|
8
|
+
# clearing :on
|
9
|
+
|
10
|
+
## Guard internally checks for changes in the Guardfile and exits.
|
11
|
+
## If you want Guard to automatically start up again, run guard in a
|
12
|
+
## shell loop, e.g.:
|
13
|
+
##
|
14
|
+
## $ while bundle exec guard; do echo "Restarting Guard..."; done
|
15
|
+
##
|
16
|
+
## Note: if you are using the `directories` clause above and you are not
|
17
|
+
## watching the project directory ('.'), then you will want to move
|
18
|
+
## the Guardfile to a watched dir and symlink it back, e.g.
|
19
|
+
#
|
20
|
+
# $ mkdir config
|
21
|
+
# $ mv Guardfile config/
|
22
|
+
# $ ln -s config/Guardfile .
|
23
|
+
#
|
24
|
+
# and, you'll have to watch "config/Guardfile" instead of "Guardfile"
|
25
|
+
|
26
|
+
# Note: The cmd option is now required due to the increasing number of ways
|
27
|
+
# rspec may be run, below are examples of the most common uses.
|
28
|
+
# * bundler: 'bundle exec rspec'
|
29
|
+
# * bundler binstubs: 'bin/rspec'
|
30
|
+
# * spring: 'bin/rspec' (This will use spring if running and you have
|
31
|
+
# installed the spring binstubs per the docs)
|
32
|
+
# * zeus: 'zeus rspec' (requires the server to be started separately)
|
33
|
+
# * 'just' rspec: 'rspec'
|
34
|
+
|
35
|
+
guard :rspec, cmd: "bundle exec rspec" do
|
36
|
+
require "guard/rspec/dsl"
|
37
|
+
dsl = Guard::RSpec::Dsl.new(self)
|
38
|
+
|
39
|
+
# Feel free to open issues for suggestions and improvements
|
40
|
+
|
41
|
+
# RSpec files
|
42
|
+
rspec = dsl.rspec
|
43
|
+
watch(rspec.spec_helper) { rspec.spec_dir }
|
44
|
+
watch(rspec.spec_support) { rspec.spec_dir }
|
45
|
+
watch(rspec.spec_files)
|
46
|
+
|
47
|
+
watch(%r{^lib/(.+)\.rb$}) { |m| ["spec/#{m[1]}_spec.rb", "spec/integration/#{m[1]}_spec.rb"] }
|
48
|
+
end
|
data/LICENSE.txt
ADDED
@@ -0,0 +1,22 @@
|
|
1
|
+
Copyright (c) 2015 Alex Leschenko
|
2
|
+
|
3
|
+
MIT License
|
4
|
+
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining
|
6
|
+
a copy of this software and associated documentation files (the
|
7
|
+
"Software"), to deal in the Software without restriction, including
|
8
|
+
without limitation the rights to use, copy, modify, merge, publish,
|
9
|
+
distribute, sublicense, and/or sell copies of the Software, and to
|
10
|
+
permit persons to whom the Software is furnished to do so, subject to
|
11
|
+
the following conditions:
|
12
|
+
|
13
|
+
The above copyright notice and this permission notice shall be
|
14
|
+
included in all copies or substantial portions of the Software.
|
15
|
+
|
16
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
17
|
+
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
18
|
+
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
19
|
+
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
|
20
|
+
LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
|
21
|
+
OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
|
22
|
+
WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
data/README.md
ADDED
@@ -0,0 +1,99 @@
|
|
1
|
+
# EsClient
|
2
|
+
|
3
|
+
This elasticsearch client is just all you need to index and search your data with persistent http connection.
|
4
|
+
It don't tend to wrap [elasticsearch](http://elasticsearch.org) dsl into ruby style dsl.
|
5
|
+
[Excon](https://github.com/excon/excon) used for http staff.
|
6
|
+
There is adapter for ActiveRecord models.
|
7
|
+
|
8
|
+
## Installation
|
9
|
+
|
10
|
+
Add this line to your application's Gemfile:
|
11
|
+
|
12
|
+
```ruby
|
13
|
+
gem 'es_client'
|
14
|
+
```
|
15
|
+
|
16
|
+
And then execute:
|
17
|
+
|
18
|
+
$ bundle
|
19
|
+
|
20
|
+
Or install it yourself as:
|
21
|
+
|
22
|
+
$ gem install es_client
|
23
|
+
|
24
|
+
## Usage
|
25
|
+
|
26
|
+
Create index:
|
27
|
+
```ruby
|
28
|
+
index = EsClient::Index.new('products', mappings: {product: {properties: {sku: {type: 'string'}}}}, settings: {number_of_shards: 1})
|
29
|
+
index.create
|
30
|
+
```
|
31
|
+
|
32
|
+
Add document to index:
|
33
|
+
```ruby
|
34
|
+
index.save_document('product', 1, {id: 1, name: 'Table', sku: '123'})
|
35
|
+
```
|
36
|
+
|
37
|
+
Fetch document:
|
38
|
+
```ruby
|
39
|
+
index.find('product', 1)
|
40
|
+
```
|
41
|
+
|
42
|
+
Add few document with one query i.e. bulk index:
|
43
|
+
```ruby
|
44
|
+
index.bulk(:index, 'product', [{id: 2, name: 'Chair'}, {id: 2, name: 'Lamp'}])
|
45
|
+
```
|
46
|
+
|
47
|
+
And, of course, search:
|
48
|
+
```ruby
|
49
|
+
index.search(query: {query_string: {query: 'table OR chair'}}).decoded
|
50
|
+
```
|
51
|
+
|
52
|
+
## With ActiveRecord model:
|
53
|
+
|
54
|
+
Include EsClient modules:
|
55
|
+
```ruby
|
56
|
+
class User
|
57
|
+
include ::EsClient::ActiveRecord::Glue
|
58
|
+
include ::EsClient::ActiveRecord::Shortcuts
|
59
|
+
end
|
60
|
+
```
|
61
|
+
|
62
|
+
Create index:
|
63
|
+
```ruby
|
64
|
+
User.es_client.index.create
|
65
|
+
```
|
66
|
+
|
67
|
+
Indexing performed on `save` and `destroy` callbacks:
|
68
|
+
```ruby
|
69
|
+
user = User.create(name: 'alex')
|
70
|
+
```
|
71
|
+
|
72
|
+
Fetch record elasticsearch document:
|
73
|
+
```ruby
|
74
|
+
user.es_doc
|
75
|
+
```
|
76
|
+
|
77
|
+
Search:
|
78
|
+
```ruby
|
79
|
+
User.es_client.search(query: {query_string: {query: 'table OR chair'}})
|
80
|
+
```
|
81
|
+
|
82
|
+
Reindex all your data:
|
83
|
+
```ruby
|
84
|
+
User.es_client_reindex
|
85
|
+
```
|
86
|
+
|
87
|
+
Or with progressbar (requires `ruby-progressbar` gem):
|
88
|
+
```ruby
|
89
|
+
User.es_client_reindex_with_progress
|
90
|
+
```
|
91
|
+
|
92
|
+
|
93
|
+
## Contributing
|
94
|
+
|
95
|
+
1. Fork it ( https://github.com/[my-github-username]/es_client/fork )
|
96
|
+
2. Create your feature branch (`git checkout -b my-new-feature`)
|
97
|
+
3. Commit your changes (`git commit -am 'Add some feature'`)
|
98
|
+
4. Push to the branch (`git push origin my-new-feature`)
|
99
|
+
5. Create a new Pull Request
|
data/ROADMAP.md
ADDED
data/Rakefile
ADDED
data/es_client.gemspec
ADDED
@@ -0,0 +1,29 @@
|
|
1
|
+
# coding: utf-8
|
2
|
+
lib = File.expand_path('../lib', __FILE__)
|
3
|
+
$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
4
|
+
require 'es_client/version'
|
5
|
+
|
6
|
+
Gem::Specification.new do |spec|
|
7
|
+
spec.name = 'es_client'
|
8
|
+
spec.version = EsClient::VERSION
|
9
|
+
spec.authors = ['Alex Leschenko']
|
10
|
+
spec.email = ['leschenko.al@gmail.com']
|
11
|
+
spec.summary = 'Simple and robust elasticsearch client'
|
12
|
+
spec.description = 'This elasticsearch client is just all you need to search and index your data with persistent http connection'
|
13
|
+
spec.homepage = 'https://github.com/leschenko/es_client'
|
14
|
+
spec.license = 'MIT'
|
15
|
+
|
16
|
+
spec.files = `git ls-files -z`.split("\x0")
|
17
|
+
spec.executables = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
|
18
|
+
spec.test_files = spec.files.grep(%r{^(test|spec|features)/})
|
19
|
+
spec.require_paths = ['lib']
|
20
|
+
|
21
|
+
spec.add_development_dependency 'bundler', '~> 1.7'
|
22
|
+
spec.add_development_dependency 'rake', '~> 10.0'
|
23
|
+
spec.add_development_dependency 'rspec'
|
24
|
+
spec.add_development_dependency 'activemodel'
|
25
|
+
spec.add_development_dependency 'ruby-progressbar'
|
26
|
+
|
27
|
+
spec.add_dependency 'excon'
|
28
|
+
spec.add_dependency 'activesupport'
|
29
|
+
end
|
data/lib/es_client.rb
ADDED
@@ -0,0 +1,45 @@
|
|
1
|
+
require 'es_client/version'
|
2
|
+
require 'excon'
|
3
|
+
require 'active_support/all'
|
4
|
+
|
5
|
+
module EsClient
|
6
|
+
autoload :Client, 'es_client/client'
|
7
|
+
autoload :Response, 'es_client/response'
|
8
|
+
autoload :Logger, 'es_client/logger'
|
9
|
+
autoload :Index, 'es_client/index'
|
10
|
+
|
11
|
+
module ActiveRecord
|
12
|
+
autoload :Adapter, 'es_client/active_record/adapter'
|
13
|
+
autoload :Glue, 'es_client/active_record/glue'
|
14
|
+
autoload :Shortcuts, 'es_client/active_record/shortcuts'
|
15
|
+
end
|
16
|
+
|
17
|
+
mattr_accessor :callbacks_enabled, :logger, :logger_options, :host, :http_client_options, :index_prefix
|
18
|
+
|
19
|
+
@@host = 'http://localhost:9200'
|
20
|
+
@@http_client_options = {persistent: true}
|
21
|
+
@@logger_options = {log_binary: true, log_response: true, pretty: true}
|
22
|
+
@@callbacks_enabled = true
|
23
|
+
|
24
|
+
def self.log_path=(path)
|
25
|
+
self.logger = ::EsClient::Logger.new(path, logger_options)
|
26
|
+
end
|
27
|
+
|
28
|
+
def self.client
|
29
|
+
@client ||= ::EsClient::Client.new(host, http_client_options)
|
30
|
+
end
|
31
|
+
|
32
|
+
def self.with_log_level(level)
|
33
|
+
old_level = logger.level
|
34
|
+
begin
|
35
|
+
self.logger.level = level.is_a?(Integer) ? level : logger.class.const_get(level.to_s.upcase)
|
36
|
+
yield
|
37
|
+
ensure
|
38
|
+
self.logger.level = old_level
|
39
|
+
end
|
40
|
+
end
|
41
|
+
|
42
|
+
def self.setup
|
43
|
+
yield self
|
44
|
+
end
|
45
|
+
end
|
@@ -0,0 +1,81 @@
|
|
1
|
+
module EsClient
|
2
|
+
module ActiveRecord
|
3
|
+
class Adapter
|
4
|
+
attr_reader :index, :index_name, :document_type, :mapping, :settings
|
5
|
+
|
6
|
+
def initialize(model)
|
7
|
+
@model = model
|
8
|
+
end
|
9
|
+
|
10
|
+
def index
|
11
|
+
@index ||= ::EsClient::Index.new(index_name)
|
12
|
+
end
|
13
|
+
|
14
|
+
def index_name(value=nil)
|
15
|
+
if value
|
16
|
+
@index_name = value
|
17
|
+
else
|
18
|
+
@index_name || @model.model_name.plural
|
19
|
+
end
|
20
|
+
end
|
21
|
+
|
22
|
+
def document_type(value=nil)
|
23
|
+
if value
|
24
|
+
@document_type = value
|
25
|
+
else
|
26
|
+
@document_type || @model.model_name.singular
|
27
|
+
end
|
28
|
+
end
|
29
|
+
|
30
|
+
def mapping(value=nil)
|
31
|
+
if value
|
32
|
+
@index.options[:mappings] ||= {}
|
33
|
+
@index.options[:mappings].deep_merge!(value)
|
34
|
+
else
|
35
|
+
index.get_mapping
|
36
|
+
end
|
37
|
+
end
|
38
|
+
|
39
|
+
def settings(value=nil)
|
40
|
+
if value
|
41
|
+
@index.options[:settings] ||= {}
|
42
|
+
@index.options[:settings].deep_merge!(value)
|
43
|
+
else
|
44
|
+
index.get_settings
|
45
|
+
end
|
46
|
+
end
|
47
|
+
|
48
|
+
def save_document(record)
|
49
|
+
index.save_document(document_type, record.id, record.as_indexed_json)
|
50
|
+
end
|
51
|
+
|
52
|
+
def update_document(record, additional_doc=nil)
|
53
|
+
doc = record.changes.map { |k, v| [k, v.last] }.to_h
|
54
|
+
doc.deep_merge!(additional_doc) if additional_doc
|
55
|
+
index.update_document(document_type, record.id, doc)
|
56
|
+
end
|
57
|
+
|
58
|
+
def destroy_document(id)
|
59
|
+
index.destroy_document(document_type, id)
|
60
|
+
end
|
61
|
+
|
62
|
+
def find(id)
|
63
|
+
if id.is_a?(Array)
|
64
|
+
query = {query: {ids: {values: id, type: document_type}}, size: id.length}
|
65
|
+
index.search(query, type: document_type)
|
66
|
+
else
|
67
|
+
index.find(document_type, id)
|
68
|
+
end
|
69
|
+
end
|
70
|
+
|
71
|
+
def import(records)
|
72
|
+
index.bulk :index, document_type, records.map(&:as_indexed_json)
|
73
|
+
end
|
74
|
+
|
75
|
+
def search(query, options={})
|
76
|
+
options[:type] = document_type
|
77
|
+
index.search(query, options)
|
78
|
+
end
|
79
|
+
end
|
80
|
+
end
|
81
|
+
end
|
@@ -0,0 +1,72 @@
|
|
1
|
+
module EsClient
|
2
|
+
module ActiveRecord
|
3
|
+
module Glue
|
4
|
+
extend ActiveSupport::Concern
|
5
|
+
|
6
|
+
included do
|
7
|
+
class_attribute :es_client
|
8
|
+
self.es_client = ::EsClient::ActiveRecord::Adapter.new(self)
|
9
|
+
after_save :es_client_save, if: :es_client_callbacks_enabled?
|
10
|
+
after_destroy :es_client_destroy, if: :es_client_callbacks_enabled?
|
11
|
+
end
|
12
|
+
|
13
|
+
def es_client_save
|
14
|
+
es_client.save_document(self)
|
15
|
+
end
|
16
|
+
|
17
|
+
def es_client_update(additional_doc=nil)
|
18
|
+
return if new_record?
|
19
|
+
es_client.update_document(self, additional_doc)
|
20
|
+
end
|
21
|
+
|
22
|
+
def es_client_destroy
|
23
|
+
return if new_record?
|
24
|
+
es_client.destroy_document(id)
|
25
|
+
end
|
26
|
+
|
27
|
+
def es_client_document(force=false)
|
28
|
+
return @es_client_document if !force && defined?(@es_client_document)
|
29
|
+
@es_client_document = es_client.find(id)
|
30
|
+
end
|
31
|
+
|
32
|
+
def es_client_callbacks_enabled?
|
33
|
+
EsClient.callbacks_enabled
|
34
|
+
end
|
35
|
+
|
36
|
+
def as_indexed_json
|
37
|
+
as_json
|
38
|
+
end
|
39
|
+
|
40
|
+
module ClassMethods
|
41
|
+
def es_client_reindex(options={})
|
42
|
+
find_in_batches(options) do |batch|
|
43
|
+
es_client.import(batch)
|
44
|
+
end
|
45
|
+
end
|
46
|
+
|
47
|
+
def es_client_reindex_with_progress(options={})
|
48
|
+
find_in_batches_with_progress(options) do |batch|
|
49
|
+
es_client.import(batch)
|
50
|
+
end
|
51
|
+
end
|
52
|
+
|
53
|
+
def find_in_batches_with_progress(options = {})
|
54
|
+
unless defined? ProgressBar
|
55
|
+
warn "Install 'ruby-progressbar' gem to use '#{__method__}' method"
|
56
|
+
return
|
57
|
+
end
|
58
|
+
options[:batch_size] ||= 1000
|
59
|
+
total = (count / options[:batch_size].to_f).ceil
|
60
|
+
title = options.delete(:name) || "#{name} batch_size:#{options[:batch_size]}"
|
61
|
+
bar = ProgressBar.create(title: title, total: total, format: '%c of %C - %a %e |%b>>%i| %p%% %t')
|
62
|
+
bar.progress_mark = '='
|
63
|
+
find_in_batches(options) do |r|
|
64
|
+
yield r
|
65
|
+
bar.increment
|
66
|
+
end
|
67
|
+
bar.finish
|
68
|
+
end unless respond_to?(:find_in_batches_with_progress)
|
69
|
+
end
|
70
|
+
end
|
71
|
+
end
|
72
|
+
end
|