es-elasticity 0.14.0 → 1.0.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.circleci/config.yml +8 -7
- data/CHANGELOG.md +13 -0
- data/Gemfile.lock +2 -2
- data/README.md +42 -38
- data/es-elasticity.gemspec +2 -2
- data/lib/elasticity/base_document.rb +3 -1
- data/lib/elasticity/bulk.rb +20 -18
- data/lib/elasticity/config.rb +2 -0
- data/lib/elasticity/document.rb +2 -0
- data/lib/elasticity/index_config.rb +19 -5
- data/lib/elasticity/index_mapper.rb +10 -9
- data/lib/elasticity/instrumented_client.rb +2 -0
- data/lib/elasticity/log_subscriber.rb +2 -0
- data/lib/elasticity/multi_search.rb +2 -0
- data/lib/elasticity/multi_search_response_parser.rb +2 -0
- data/lib/elasticity/railtie.rb +3 -1
- data/lib/elasticity/search.rb +6 -4
- data/lib/elasticity/segmented_document.rb +2 -0
- data/lib/elasticity/strategies/alias_index.rb +28 -32
- data/lib/elasticity/strategies/single_index.rb +16 -20
- data/lib/elasticity/strategies.rb +2 -0
- data/lib/elasticity/version.rb +3 -1
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: a71a1d59a0baa8af334a40cc3b38c8210b3e962e9daa0a899820bad2d1a2598e
|
4
|
+
data.tar.gz: 2c7032d5508738f6588c252e126d5213453204359f2bf8cae242359c1a36e7ec
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 5892bd0e785358bf7d8bd9b3946d3c8561549aad4c95adde310412bddf340e06413db60aa2cd5ebcc725b7cef1bb946d677918de4d323d1b0ca2f4d6873a329e
|
7
|
+
data.tar.gz: 53478a2feb259f44b52ab87738abbb670a292efa3782e9a41302f0a015e2a1c1d08b9e3fe73c0e67075f9d78f0daab17aa7ad0e37b4e91b8bf4945b4b0b86e1a
|
data/.circleci/config.yml
CHANGED
@@ -79,12 +79,12 @@ jobs:
|
|
79
79
|
gem install bundler --version "${BUNDLE_VERSION}" --force
|
80
80
|
- restore_cache:
|
81
81
|
keys:
|
82
|
-
-
|
82
|
+
- v3-bundle-ruby_2_7-{{ checksum "Gemfile.lock" }}
|
83
83
|
- run:
|
84
84
|
name: Install Ruby Dependencies
|
85
85
|
command: bundle check --path=vendor/bundle || bundle install --local --frozen --path=vendor/bundle --jobs=4 --retry=3
|
86
86
|
- save_cache:
|
87
|
-
key:
|
87
|
+
key: v3-bundle-ruby_2_7-{{ checksum "Gemfile.lock" }}
|
88
88
|
paths:
|
89
89
|
- vendor/bundle
|
90
90
|
- run:
|
@@ -107,12 +107,12 @@ jobs:
|
|
107
107
|
gem install bundler --version "${BUNDLE_VERSION}" --force
|
108
108
|
- restore_cache:
|
109
109
|
keys:
|
110
|
-
-
|
110
|
+
- v3-bundle-ruby_3_0-{{ checksum "Gemfile.lock" }}
|
111
111
|
- run:
|
112
112
|
name: Install Ruby Dependencies
|
113
113
|
command: bundle check --path=vendor/bundle || bundle install --local --frozen --path=vendor/bundle --jobs=4 --retry=3
|
114
114
|
- save_cache:
|
115
|
-
key:
|
115
|
+
key: v3-bundle-ruby_3_0-{{ checksum "Gemfile.lock" }}
|
116
116
|
paths:
|
117
117
|
- vendor/bundle
|
118
118
|
- run:
|
@@ -136,12 +136,12 @@ jobs:
|
|
136
136
|
gem install bundler --version "${BUNDLE_VERSION}" --force
|
137
137
|
- restore_cache:
|
138
138
|
keys:
|
139
|
-
-
|
139
|
+
- v3-bundle-ruby_3_1-{{ checksum "Gemfile.lock" }}
|
140
140
|
- run:
|
141
141
|
name: Install Ruby Dependencies
|
142
142
|
command: bundle check --path=vendor/bundle || bundle install --local --frozen --path=vendor/bundle --jobs=4 --retry=3
|
143
143
|
- save_cache:
|
144
|
-
key:
|
144
|
+
key: v3-bundle-ruby_3_1-{{ checksum "Gemfile.lock" }}
|
145
145
|
paths:
|
146
146
|
- vendor/bundle
|
147
147
|
- run:
|
@@ -194,7 +194,8 @@ workflows:
|
|
194
194
|
- gem/publish:
|
195
195
|
<<: *pr_only
|
196
196
|
name: gem-publish
|
197
|
-
to_rubygems:
|
197
|
+
to_rubygems: false
|
198
|
+
to_nexus: true
|
198
199
|
pre_release: true
|
199
200
|
requires:
|
200
201
|
- pre-release-approval
|
data/CHANGELOG.md
CHANGED
@@ -6,6 +6,19 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
|
|
6
6
|
|
7
7
|
## [Unreleased]
|
8
8
|
|
9
|
+
## [1.0.0] - 2022-08-09
|
10
|
+
### Changed
|
11
|
+
- Consistently use quotations
|
12
|
+
- Update API to remove deprecation warnings for Elasticsearch 7.X.X in preparation for upgrade to 8.
|
13
|
+
- Freeze string literals
|
14
|
+
- Remove `_type` from query/create/update/delete methods
|
15
|
+
- Remove `use_new_timestamp_format` config
|
16
|
+
- Remove `include_type_name_on_create` config
|
17
|
+
|
18
|
+
## [0.14.1] - 2022-08-09
|
19
|
+
### Changed
|
20
|
+
- Don't publish pre-release builds to RubyGems
|
21
|
+
|
9
22
|
## [0.14.0] - 2022-08-04
|
10
23
|
### Changed
|
11
24
|
- Use `refresh` instead of `flush` where search is expected to be updated
|
data/Gemfile.lock
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
PATH
|
2
2
|
remote: .
|
3
3
|
specs:
|
4
|
-
es-elasticity (0.
|
4
|
+
es-elasticity (1.0.0)
|
5
5
|
activemodel (>= 5.2.0, < 7.1)
|
6
6
|
activerecord (>= 5.2.0, < 7.1)
|
7
7
|
activesupport (>= 5.2.0, < 7.1)
|
@@ -34,7 +34,7 @@ GEM
|
|
34
34
|
elasticsearch-transport (7.17.1)
|
35
35
|
faraday (~> 1)
|
36
36
|
multi_json
|
37
|
-
faraday (1.10.
|
37
|
+
faraday (1.10.1)
|
38
38
|
faraday-em_http (~> 1.0)
|
39
39
|
faraday-em_synchrony (~> 1.0)
|
40
40
|
faraday-excon (~> 1.1)
|
data/README.md
CHANGED
@@ -29,9 +29,10 @@ Or install it yourself as:
|
|
29
29
|
## Usage
|
30
30
|
|
31
31
|
### Version Support
|
32
|
-
This gem has [elasticsearch-ruby](https://github.com/elastic/elasticsearch-ruby) as a dependency. In order to use different versions of elasticsearch you will need to match your version of elasticsearch-ruby to the version of elasticsearch you want to use ([see here](https://github.com/elastic/elasticsearch-ruby#compatibility). Elasticity should work across all versions of elastisearch-ruby, although they have not all been tested so there are likely edge cases.
|
33
32
|
|
34
|
-
|
33
|
+
This gem has [elasticsearch-ruby](https://github.com/elastic/elasticsearch-ruby) as a dependency. In order to use different versions of elasticsearch you will need to match your version of elasticsearch-ruby to the version of elasticsearch you want to use ([see here](https://github.com/elastic/elasticsearch-ruby#compatibility). Elasticity should work across all versions of elastisearch-ruby, although they have not all been tested so there are likely edge cases.
|
34
|
+
|
35
|
+
Currently tests are run on CirlceCI against elasticsearch 7.17.3 with elasticsearch-ruby 7.17.1.
|
35
36
|
|
36
37
|
### Configuration
|
37
38
|
|
@@ -149,7 +150,6 @@ documents = [
|
|
149
150
|
Search::User.bulk_update(documents)
|
150
151
|
```
|
151
152
|
|
152
|
-
|
153
153
|
### Searching
|
154
154
|
|
155
155
|
Class methods have access to the `search` method, which returns a lazy evaluated search. That means that the search will only be performed when the data is necessary, not when the `search` method is called.
|
@@ -194,7 +194,8 @@ adults = adults.active_records(User)
|
|
194
194
|
#### Search Args
|
195
195
|
|
196
196
|
##### explain: true
|
197
|
-
|
197
|
+
|
198
|
+
For `search` definitions we support passing `{ explain: true }` to the search as a second argument in order to surface the reason a search result was returned.
|
198
199
|
|
199
200
|
```ruby
|
200
201
|
# example in single search
|
@@ -211,6 +212,7 @@ multi = Elasticity::MultiSearch.new do |m|
|
|
211
212
|
m.add(:c, search_c, documents: ::SearchDoc::C)
|
212
213
|
end
|
213
214
|
```
|
215
|
+
|
214
216
|
For more information about the `active_records` method, read [ActiveRecord integration](#activerecord-integration).
|
215
217
|
|
216
218
|
### Segmented Documents
|
@@ -324,15 +326,15 @@ When the mapping needs to change, a hot remapping can be performed by doing the
|
|
324
326
|
1. Create a new index with the new mapping;
|
325
327
|
2. change the update alias to point to the new index, and change main alias to point to both indexes; at this point it will look something like this:
|
326
328
|
|
327
|
-
|
328
|
-
|
329
|
-
|
330
|
-
|
331
|
-
|
332
|
-
|
333
|
-
|
334
|
-
|
335
|
-
|
329
|
+
```
|
330
|
+
|¯¯¯¯¯¯¯¯¯¯¯¯¯|----------------------> |¯¯¯¯¯¯¯¯¯¯¯¯¯|
|
331
|
+
| MainAlias | | Old Index |
|
332
|
+
|_____________|----------| |_____________|
|
333
|
+
|
|
334
|
+
|¯¯¯¯¯¯¯¯¯¯¯¯¯| |-----------> |¯¯¯¯¯¯¯¯¯¯¯¯¯|
|
335
|
+
| UpdateAlias |----------------------> | New Index |
|
336
|
+
|_____________| |_____________|
|
337
|
+
```
|
336
338
|
|
337
339
|
3. iterate over all documents on the old index, copying them to the new index;
|
338
340
|
4. change aliases to point only to the new index;
|
@@ -344,43 +346,43 @@ This is a simplified version, there are other things that happen to ensure consi
|
|
344
346
|
|
345
347
|
ActiveRecord integration is mainly a set of conventions rather than implementation, with the exception of one method that allows mapping documents back to a relation. Here is the list of conventions:
|
346
348
|
|
347
|
-
|
348
|
-
|
349
|
-
|
349
|
+
- have a class method on the document called `from_active_record` that creates a document object from the active record object;
|
350
|
+
- have a class method on the Document for rebuilding the index from the records;
|
351
|
+
- have an `after_save` and an `after_destroy` callbacks on the ActiveRecord model;
|
350
352
|
|
351
353
|
For example:
|
352
354
|
|
353
|
-
|
354
|
-
|
355
|
-
|
356
|
-
|
355
|
+
```ruby
|
356
|
+
class User < ActiveRecord::Base
|
357
|
+
after_save :update_index_document
|
358
|
+
after_destroy :delete_index_document
|
357
359
|
|
358
|
-
|
359
|
-
|
360
|
-
|
360
|
+
def update_index_document
|
361
|
+
Search::User.from_active_record(self).update
|
362
|
+
end
|
361
363
|
|
362
|
-
|
363
|
-
|
364
|
-
end
|
364
|
+
def delete_index_document
|
365
|
+
Search::User.delete(self.id)
|
365
366
|
end
|
367
|
+
end
|
366
368
|
|
367
|
-
|
368
|
-
|
369
|
+
class Search::User < Elasticity::Document
|
370
|
+
# ... configuration
|
369
371
|
|
370
|
-
|
371
|
-
|
372
|
-
|
372
|
+
def self.from_active_record(ar)
|
373
|
+
new(name: ar.name, birthdate: ar.birthdate)
|
374
|
+
end
|
373
375
|
|
374
|
-
|
375
|
-
|
376
|
+
def self.rebuild_index
|
377
|
+
self.recreate_index
|
376
378
|
|
377
|
-
|
378
|
-
|
379
|
-
|
380
|
-
end
|
379
|
+
User.find_in_batches do |batch|
|
380
|
+
documents = batch.map { |record| from_active_record(record) }
|
381
|
+
self.bulk_index(documents)
|
381
382
|
end
|
382
383
|
end
|
383
|
-
|
384
|
+
end
|
385
|
+
```
|
384
386
|
|
385
387
|
This makes the code very clear in intent, easier to see when and how things happen and under the developer control, keeping both parts very decoupled.
|
386
388
|
|
@@ -417,6 +419,7 @@ Search::User.adults.active_records(User.where(active: true))
|
|
417
419
|
```
|
418
420
|
|
419
421
|
## Upgrading from 0.7.0 to 0.8.0
|
422
|
+
|
420
423
|
The default persistence strategy changed from SingleIndex to AliasIndex in version 0.8.0 Add the following to your Document configuration to maintain the legacy behaviour.
|
421
424
|
|
422
425
|
```ruby
|
@@ -424,6 +427,7 @@ The default persistence strategy changed from SingleIndex to AliasIndex in versi
|
|
424
427
|
```
|
425
428
|
|
426
429
|
## Roadmap
|
430
|
+
|
427
431
|
- [ ] Define from_active_record interface
|
428
432
|
- [ ] Write more detailed documentation section for:
|
429
433
|
- [ ] Model definition
|
data/es-elasticity.gemspec
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
# coding: utf-8
|
2
|
-
lib = File.expand_path(
|
2
|
+
lib = File.expand_path("../lib", __FILE__)
|
3
3
|
$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
4
|
-
require
|
4
|
+
require "elasticity/version"
|
5
5
|
|
6
6
|
Gem::Specification.new do |spec|
|
7
7
|
spec.name = "es-elasticity"
|
@@ -1,3 +1,5 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
module Elasticity
|
2
4
|
class BaseDocument
|
3
5
|
include ::ActiveModel::Model
|
@@ -60,7 +62,7 @@ module Elasticity
|
|
60
62
|
end
|
61
63
|
|
62
64
|
def self.default_document_type
|
63
|
-
self.name.gsub(
|
65
|
+
self.name.gsub("::", "_").underscore
|
64
66
|
end
|
65
67
|
|
66
68
|
def self.default_index_base_name
|
data/lib/elasticity/bulk.rb
CHANGED
@@ -1,3 +1,5 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
module Elasticity
|
2
4
|
class Bulk
|
3
5
|
def initialize(client)
|
@@ -5,16 +7,16 @@ module Elasticity
|
|
5
7
|
@operations = []
|
6
8
|
end
|
7
9
|
|
8
|
-
def index(index_name,
|
9
|
-
@operations << { index: { _index: index_name,
|
10
|
+
def index(index_name, id, attributes)
|
11
|
+
@operations << { index: { _index: index_name, _id: id, data: attributes }}
|
10
12
|
end
|
11
13
|
|
12
|
-
def update(index_name,
|
13
|
-
@operations << { update: { _index: index_name,
|
14
|
+
def update(index_name, id, attributes)
|
15
|
+
@operations << { update: { _index: index_name, _id: id, data: attributes }}
|
14
16
|
end
|
15
17
|
|
16
|
-
def delete(index_name,
|
17
|
-
@operations << { delete: { _index: index_name,
|
18
|
+
def delete(index_name, id)
|
19
|
+
@operations << { delete: { _index: index_name, _id: id }}
|
18
20
|
end
|
19
21
|
|
20
22
|
def execute
|
@@ -27,16 +29,16 @@ module Elasticity
|
|
27
29
|
@index_name = index_name
|
28
30
|
end
|
29
31
|
|
30
|
-
def index(
|
31
|
-
super(@index_name,
|
32
|
+
def index(id, attributes)
|
33
|
+
super(@index_name, id, attributes)
|
32
34
|
end
|
33
35
|
|
34
|
-
def update(
|
35
|
-
super(@index_name,
|
36
|
+
def update(id, attributes)
|
37
|
+
super(@index_name, id, attributes)
|
36
38
|
end
|
37
39
|
|
38
|
-
def delete(
|
39
|
-
super(@index_name,
|
40
|
+
def delete(id)
|
41
|
+
super(@index_name, id)
|
40
42
|
end
|
41
43
|
end
|
42
44
|
|
@@ -47,17 +49,17 @@ module Elasticity
|
|
47
49
|
@delete_indexes = delete_indexes
|
48
50
|
end
|
49
51
|
|
50
|
-
def index(
|
51
|
-
super(@update_alias,
|
52
|
+
def index(id, attributes)
|
53
|
+
super(@update_alias, id, attributes)
|
52
54
|
end
|
53
55
|
|
54
|
-
def update(
|
55
|
-
super(@update_alias,
|
56
|
+
def update(id, attributes)
|
57
|
+
super(@update_alias, id, attributes)
|
56
58
|
end
|
57
59
|
|
58
|
-
def delete(
|
60
|
+
def delete(id)
|
59
61
|
@delete_indexes.each do |index|
|
60
|
-
super(index,
|
62
|
+
super(index, id)
|
61
63
|
end
|
62
64
|
end
|
63
65
|
end
|
data/lib/elasticity/config.rb
CHANGED
data/lib/elasticity/document.rb
CHANGED
@@ -1,18 +1,21 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
module Elasticity
|
2
4
|
class IndexConfig
|
3
5
|
class SubclassError < StandardError; end
|
4
6
|
|
5
|
-
SUBCLASSES_WARNING = "Indices created in Elasticsearch 6.0.0 or later may only contain a single mapping type. "
|
7
|
+
SUBCLASSES_WARNING = "Indices created in Elasticsearch 6.0.0 or later may only contain a single mapping type. "\
|
6
8
|
"Therefore, doument-type based inheritance has been disabled by Elasticity"
|
7
|
-
SUBCLASSES_ERROR = "Mapping types have been completely removed in Elasticsearch 7.0.0. "
|
9
|
+
SUBCLASSES_ERROR = "Mapping types have been completely removed in Elasticsearch 7.0.0. "\
|
8
10
|
"Therefore, doument-type based inheritance has been disabled by Elasticity"
|
9
|
-
VERSION_FOR_SUBCLASS_WARNING = "6.0.0"
|
10
|
-
VERSION_FOR_SUBCLASS_ERROR = "7.0.0"
|
11
|
+
VERSION_FOR_SUBCLASS_WARNING = "6.0.0"
|
12
|
+
VERSION_FOR_SUBCLASS_ERROR = "7.0.0"
|
11
13
|
ATTRS = [
|
12
14
|
:index_base_name, :document_type, :mapping, :strategy, :subclasses,
|
13
15
|
:settings, :use_new_timestamp_format, :include_type_name_on_create
|
14
16
|
].freeze
|
15
17
|
VALIDATABLE_ATTRS = [:index_base_name, :document_type, :strategy].freeze
|
18
|
+
DEPRECATED_ATTRS = [:use_new_timestamp_format, :include_type_name_on_create].freeze
|
16
19
|
|
17
20
|
attr_accessor(*ATTRS)
|
18
21
|
|
@@ -23,6 +26,7 @@ module Elasticity
|
|
23
26
|
@elasticity_config = elasticity_config
|
24
27
|
yield(self)
|
25
28
|
subclasses_warning_or_exception
|
29
|
+
warn_deprecated_config
|
26
30
|
validate!
|
27
31
|
end
|
28
32
|
|
@@ -38,9 +42,10 @@ module Elasticity
|
|
38
42
|
|
39
43
|
def definition
|
40
44
|
return @definition if defined?(@definition)
|
45
|
+
|
41
46
|
@definition = {
|
42
47
|
settings: merge_settings,
|
43
|
-
mappings:
|
48
|
+
mappings: @mapping.nil? ? {} : @mapping.deep_stringify_keys
|
44
49
|
}
|
45
50
|
subclasses.each do |doc_type, subclass|
|
46
51
|
@definition[:mappings][doc_type] = subclass.constantize.mapping
|
@@ -80,6 +85,15 @@ module Elasticity
|
|
80
85
|
@elasticity_config.settings.merge(settings || {})
|
81
86
|
end
|
82
87
|
|
88
|
+
def warn_deprecated_config
|
89
|
+
DEPRECATED_ATTRS.each do |attr|
|
90
|
+
ActiveSupport::Deprecation.warn(
|
91
|
+
"#{attr} is deprecated and will be "\
|
92
|
+
"removed in the next major release."
|
93
|
+
) if public_send(attr).present?
|
94
|
+
end
|
95
|
+
end
|
96
|
+
|
83
97
|
def subclasses_warning_or_exception
|
84
98
|
return if subclasses.nil? || subclasses.empty?
|
85
99
|
raise(SubclassError.new(SUBCLASSES_ERROR)) if es_version_meets_or_exceeds?(VERSION_FOR_SUBCLASS_ERROR)
|
@@ -1,3 +1,5 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
module Elasticity
|
2
4
|
class IndexMapper
|
3
5
|
def self.set_delegates(obj, to)
|
@@ -84,7 +86,7 @@ module Elasticity
|
|
84
86
|
|
85
87
|
# Index the given document
|
86
88
|
def index_document(id, document_hash)
|
87
|
-
@strategy.index_document(
|
89
|
+
@strategy.index_document(id, document_hash)
|
88
90
|
end
|
89
91
|
|
90
92
|
# Searches the index using the parameters provided in the body hash, following the same
|
@@ -99,25 +101,25 @@ module Elasticity
|
|
99
101
|
|
100
102
|
# Fetches one specific document from the index by ID.
|
101
103
|
def get(id)
|
102
|
-
doc = @strategy.get_document(
|
103
|
-
@document_klass.new(doc["_source"].merge(_id: doc[
|
104
|
+
doc = @strategy.get_document(id)
|
105
|
+
@document_klass.new(doc["_source"].merge(_id: doc["_id"])) if doc.present?
|
104
106
|
end
|
105
107
|
|
106
108
|
# Removes one specific document from the index.
|
107
109
|
def delete(id)
|
108
|
-
@strategy.delete_document(
|
110
|
+
@strategy.delete_document(id)
|
109
111
|
end
|
110
112
|
|
111
113
|
# Removes entries based on a search
|
112
114
|
def delete_by_search(search)
|
113
|
-
@strategy.delete_by_query(
|
115
|
+
@strategy.delete_by_query(search.body)
|
114
116
|
end
|
115
117
|
|
116
118
|
# Bulk index the provided documents
|
117
119
|
def bulk_index(documents)
|
118
120
|
@strategy.bulk do |b|
|
119
121
|
documents.each do |doc|
|
120
|
-
b.index(
|
122
|
+
b.index(doc._id, doc.to_document)
|
121
123
|
end
|
122
124
|
end
|
123
125
|
end
|
@@ -127,7 +129,6 @@ module Elasticity
|
|
127
129
|
@strategy.bulk do |b|
|
128
130
|
documents.each do |doc|
|
129
131
|
b.update(
|
130
|
-
document_type,
|
131
132
|
doc[:_id],
|
132
133
|
{ doc: { doc[:attr_name] => doc[:attr_value] } }
|
133
134
|
)
|
@@ -139,7 +140,7 @@ module Elasticity
|
|
139
140
|
def bulk_delete(ids)
|
140
141
|
@strategy.bulk do |b|
|
141
142
|
ids.each do |id|
|
142
|
-
b.delete(
|
143
|
+
b.delete(id)
|
143
144
|
end
|
144
145
|
end
|
145
146
|
end
|
@@ -156,7 +157,7 @@ module Elasticity
|
|
156
157
|
|
157
158
|
if hit["highlight"]
|
158
159
|
highlighted_attrs = hit["highlight"].each_with_object({}) do |(name, v), attrs|
|
159
|
-
name = name.gsub(/\..*\z/,
|
160
|
+
name = name.gsub(/\..*\z/, "")
|
160
161
|
|
161
162
|
attrs[name] ||= v
|
162
163
|
end
|
data/lib/elasticity/railtie.rb
CHANGED
@@ -1,8 +1,10 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
require "elasticity/log_subscriber"
|
2
4
|
|
3
5
|
module Elasticity
|
4
6
|
class Railtie < Rails::Railtie
|
5
|
-
initializer
|
7
|
+
initializer "elasticity.initialize_logging" do
|
6
8
|
LogSubscriber.attach_to(:elasticity)
|
7
9
|
end
|
8
10
|
end
|
data/lib/elasticity/search.rb
CHANGED
@@ -1,3 +1,5 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
module Elasticity
|
2
4
|
module Search
|
3
5
|
def self.build(client, index_name, document_types, body, search_args = {})
|
@@ -6,7 +8,7 @@ module Elasticity
|
|
6
8
|
end
|
7
9
|
|
8
10
|
# Elasticity::Search::Definition is a struct that encapsulates all the data specific to one
|
9
|
-
#
|
11
|
+
# Elasticsearch search.
|
10
12
|
class Definition
|
11
13
|
attr_accessor :index_name, :document_types, :body
|
12
14
|
|
@@ -22,20 +24,20 @@ module Elasticity
|
|
22
24
|
end
|
23
25
|
|
24
26
|
def to_count_args
|
25
|
-
{ index: @index_name
|
27
|
+
{ index: @index_name }.tap do |args|
|
26
28
|
body = @body.slice(:query)
|
27
29
|
args[:body] = body if body.present?
|
28
30
|
end
|
29
31
|
end
|
30
32
|
|
31
33
|
def to_search_args
|
32
|
-
@search_args.merge({ index: @index_name,
|
34
|
+
@search_args.merge({ index: @index_name, body: @body })
|
33
35
|
end
|
34
36
|
|
35
37
|
def to_msearch_args
|
36
38
|
search_body = @search_args.merge(@body)
|
37
39
|
|
38
|
-
{ index: @index_name,
|
40
|
+
{ index: @index_name, search: search_body }
|
39
41
|
end
|
40
42
|
end
|
41
43
|
|
@@ -1,27 +1,23 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
module Elasticity
|
2
4
|
module Strategies
|
3
5
|
# This strategy keeps two aliases that might be mapped to the same index or different index, allowing
|
4
6
|
# runtime changes by simply atomically updating the aliases. For example, look at the remap method
|
5
7
|
# implementation.
|
6
8
|
class AliasIndex
|
7
|
-
SNAPSHOT_ERROR_SNIPPET = "Cannot delete indices that are being snapshotted"
|
9
|
+
SNAPSHOT_ERROR_SNIPPET = "Cannot delete indices that are being snapshotted"
|
8
10
|
RETRYABLE_ERROR_SNIPPETS = [
|
9
11
|
SNAPSHOT_ERROR_SNIPPET
|
10
12
|
].freeze
|
11
13
|
|
12
14
|
STATUSES = [:missing, :ok]
|
13
15
|
|
14
|
-
def initialize(client, index_base_name, document_type, use_new_timestamp_format =
|
16
|
+
def initialize(client, index_base_name, document_type, use_new_timestamp_format = nil, include_type_name_on_create = nil)
|
15
17
|
@client = client
|
16
18
|
@main_alias = index_base_name
|
17
19
|
@update_alias = "#{index_base_name}_update"
|
18
20
|
@document_type = document_type
|
19
|
-
|
20
|
-
# Deprecated: The use_new_timestamp_format option is no longer used and will be removed in the next version.
|
21
|
-
@use_new_timestamp_format = use_new_timestamp_format
|
22
|
-
|
23
|
-
# included for compatibility with v7
|
24
|
-
@include_type_name_on_create = include_type_name_on_create
|
25
21
|
end
|
26
22
|
|
27
23
|
def ref_index_name
|
@@ -63,22 +59,22 @@ module Elasticity
|
|
63
59
|
})
|
64
60
|
|
65
61
|
@client.index_refresh(index: original_index)
|
66
|
-
cursor = @client.search index: original_index, search_type: :query_then_fetch, scroll:
|
62
|
+
cursor = @client.search index: original_index, search_type: :query_then_fetch, scroll: "10m", size: 100
|
67
63
|
loop do
|
68
|
-
hits = cursor[
|
64
|
+
hits = cursor["hits"]["hits"]
|
69
65
|
break if hits.empty?
|
70
66
|
|
71
67
|
# Fetch documents based on the ids that existed when the migration started, to make sure we only migrate
|
72
68
|
# documents that haven't been deleted.
|
73
69
|
id_docs = hits.map do |hit|
|
74
|
-
{ _index: original_index,
|
70
|
+
{ _index: original_index, _id: hit["_id"] }
|
75
71
|
end
|
76
72
|
|
77
73
|
docs = @client.mget(body: { docs: id_docs }, refresh: true)["docs"]
|
78
74
|
break if docs.empty?
|
79
75
|
|
80
76
|
# Modify document hashes to match the mapping definition so that legacy fields aren't added
|
81
|
-
defined_mapping_fields = index_def[:mappings][
|
77
|
+
defined_mapping_fields = index_def[:mappings]["properties"].keys
|
82
78
|
|
83
79
|
# Move only documents that still exists on the old index, into the new index.
|
84
80
|
ops = []
|
@@ -86,7 +82,7 @@ module Elasticity
|
|
86
82
|
if doc["found"]
|
87
83
|
legacy_fields = doc["_source"].keys - defined_mapping_fields
|
88
84
|
legacy_fields.each { |field| doc["_source"].delete(field) }
|
89
|
-
ops << { index: { _index: new_index,
|
85
|
+
ops << { index: { _index: new_index, _id: doc["_id"], data: doc["_source"] } }
|
90
86
|
end
|
91
87
|
end
|
92
88
|
|
@@ -96,12 +92,12 @@ module Elasticity
|
|
96
92
|
ops = []
|
97
93
|
@client.mget(body: { docs: id_docs }, refresh: true)["docs"].each_with_index do |new_doc, idx|
|
98
94
|
if docs[idx]["found"] && !new_doc["found"]
|
99
|
-
ops << { delete: { _index: new_index,
|
95
|
+
ops << { delete: { _index: new_index, _id: new_doc["_id"] } }
|
100
96
|
end
|
101
97
|
end
|
102
98
|
|
103
99
|
@client.bulk(body: ops) unless ops.empty?
|
104
|
-
cursor = @client.scroll(scroll_id: cursor[
|
100
|
+
cursor = @client.scroll(scroll_id: cursor["_scroll_id"], scroll: "1m", body: { scroll_id: cursor["_scroll_id"] })
|
105
101
|
end
|
106
102
|
|
107
103
|
# Update aliases to only point to the new index.
|
@@ -133,19 +129,19 @@ module Elasticity
|
|
133
129
|
})
|
134
130
|
|
135
131
|
@client.index_refresh(index: new_index)
|
136
|
-
cursor = @client.search index: new_index, search_type: :query_then_fetch, scroll:
|
132
|
+
cursor = @client.search index: new_index, search_type: :query_then_fetch, scroll: "1m", size: 100
|
137
133
|
loop do
|
138
|
-
hits = cursor[
|
134
|
+
hits = cursor["hits"]["hits"]
|
139
135
|
break if hits.empty?
|
140
136
|
|
141
137
|
# Move all the documents that exists on the new index back to the old index
|
142
138
|
ops = []
|
143
139
|
hits.each do |doc|
|
144
|
-
ops << { index: { _index: original_index,
|
140
|
+
ops << { index: { _index: original_index, _id: doc["_id"], data: doc["_source"] } }
|
145
141
|
end
|
146
142
|
|
147
143
|
@client.bulk(body: ops)
|
148
|
-
cursor = @client.scroll(scroll_id: cursor[
|
144
|
+
cursor = @client.scroll(scroll_id: cursor["_scroll_id"], scroll: "1m")
|
149
145
|
end
|
150
146
|
|
151
147
|
@client.index_refresh(index: original_index)
|
@@ -224,8 +220,8 @@ module Elasticity
|
|
224
220
|
create(index_def)
|
225
221
|
end
|
226
222
|
|
227
|
-
def index_document(
|
228
|
-
res = @client.index(index: @update_alias,
|
223
|
+
def index_document(id, attributes)
|
224
|
+
res = @client.index(index: @update_alias, id: id, body: attributes)
|
229
225
|
|
230
226
|
if id = res["_id"]
|
231
227
|
[id, res["_shards"] && res["_shards"]["successful"].to_i > 0]
|
@@ -234,24 +230,24 @@ module Elasticity
|
|
234
230
|
end
|
235
231
|
end
|
236
232
|
|
237
|
-
def delete_document(
|
233
|
+
def delete_document(id)
|
238
234
|
ops = (main_indexes | update_indexes).map do |index|
|
239
|
-
{ delete: { _index: index,
|
235
|
+
{ delete: { _index: index, _id: id } }
|
240
236
|
end
|
241
237
|
|
242
238
|
@client.bulk(body: ops)
|
243
239
|
end
|
244
240
|
|
245
|
-
def get_document(
|
246
|
-
@client.get(index: @main_alias,
|
241
|
+
def get_document(id)
|
242
|
+
@client.get(index: @main_alias, id: id)
|
247
243
|
end
|
248
244
|
|
249
245
|
def search_index
|
250
246
|
@main_alias
|
251
247
|
end
|
252
248
|
|
253
|
-
def delete_by_query(
|
254
|
-
@client.delete_by_query(index: @main_alias,
|
249
|
+
def delete_by_query(body)
|
250
|
+
@client.delete_by_query(index: @main_alias, body: body)
|
255
251
|
end
|
256
252
|
|
257
253
|
def bulk
|
@@ -269,21 +265,21 @@ module Elasticity
|
|
269
265
|
end
|
270
266
|
|
271
267
|
def settings
|
272
|
-
@client.index_get_settings(index: @main_alias
|
268
|
+
@client.index_get_settings(index: @main_alias).values.first
|
273
269
|
rescue Elasticsearch::Transport::Transport::Errors::NotFound
|
274
270
|
nil
|
275
271
|
end
|
276
272
|
|
277
273
|
def mappings
|
278
274
|
ActiveSupport::Deprecation.warn(
|
279
|
-
|
280
|
-
|
275
|
+
"Elasticity::Strategies::AliasIndex#mappings is deprecated, "\
|
276
|
+
"use mapping instead"
|
281
277
|
)
|
282
278
|
mapping
|
283
279
|
end
|
284
280
|
|
285
281
|
def mapping
|
286
|
-
@client.index_get_mapping(index: @main_alias
|
282
|
+
@client.index_get_mapping(index: @main_alias).values.first
|
287
283
|
rescue Elasticsearch::Transport::Transport::Errors::NotFound
|
288
284
|
nil
|
289
285
|
end
|
@@ -297,7 +293,7 @@ module Elasticity
|
|
297
293
|
|
298
294
|
def create_index(index_def)
|
299
295
|
name = build_index_name
|
300
|
-
@client.index_create(index: name, body: index_def
|
296
|
+
@client.index_create(index: name, body: index_def)
|
301
297
|
name
|
302
298
|
end
|
303
299
|
|
@@ -1,18 +1,14 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
module Elasticity
|
2
4
|
module Strategies
|
3
5
|
class SingleIndex
|
4
6
|
STATUSES = [:missing, :ok]
|
5
7
|
|
6
|
-
def initialize(client, index_name, document_type, use_new_timestamp_format =
|
8
|
+
def initialize(client, index_name, document_type, use_new_timestamp_format = nil, include_type_name_on_create = nil)
|
7
9
|
@client = client
|
8
10
|
@index_name = index_name
|
9
11
|
@document_type = document_type
|
10
|
-
|
11
|
-
# included for compatibility with v7
|
12
|
-
@include_type_name_on_create = include_type_name_on_create
|
13
|
-
|
14
|
-
# Deprecated: The use_new_timestamp_format option is no longer used and will be removed in the next version.
|
15
|
-
@use_new_timestamp_format = use_new_timestamp_format
|
16
12
|
end
|
17
13
|
|
18
14
|
def ref_index_name
|
@@ -29,7 +25,7 @@ module Elasticity
|
|
29
25
|
|
30
26
|
def create(index_def)
|
31
27
|
if missing?
|
32
|
-
@client.index_create(index: @index_name, body: index_def
|
28
|
+
@client.index_create(index: @index_name, body: index_def)
|
33
29
|
else
|
34
30
|
raise IndexError.new(@index_name, "index already exist")
|
35
31
|
end
|
@@ -52,8 +48,8 @@ module Elasticity
|
|
52
48
|
create(index_def)
|
53
49
|
end
|
54
50
|
|
55
|
-
def index_document(
|
56
|
-
res = @client.index(index: @index_name,
|
51
|
+
def index_document(id, attributes)
|
52
|
+
res = @client.index(index: @index_name, id: id, body: attributes)
|
57
53
|
|
58
54
|
if id = res["_id"]
|
59
55
|
[id, res["created"]]
|
@@ -62,20 +58,20 @@ module Elasticity
|
|
62
58
|
end
|
63
59
|
end
|
64
60
|
|
65
|
-
def delete_document(
|
66
|
-
@client.delete(index: @index_name,
|
61
|
+
def delete_document(id)
|
62
|
+
@client.delete(index: @index_name, id: id)
|
67
63
|
end
|
68
64
|
|
69
|
-
def get_document(
|
70
|
-
@client.get(index: @index_name,
|
65
|
+
def get_document(id)
|
66
|
+
@client.get(index: @index_name, id: id)
|
71
67
|
end
|
72
68
|
|
73
69
|
def search_index
|
74
70
|
@index_name
|
75
71
|
end
|
76
72
|
|
77
|
-
def delete_by_query(
|
78
|
-
@client.delete_by_query(index: @index_name,
|
73
|
+
def delete_by_query(body)
|
74
|
+
@client.delete_by_query(index: @index_name, body: body)
|
79
75
|
end
|
80
76
|
|
81
77
|
def bulk
|
@@ -85,21 +81,21 @@ module Elasticity
|
|
85
81
|
end
|
86
82
|
|
87
83
|
def settings
|
88
|
-
@client.index_get_settings(index: @index_name
|
84
|
+
@client.index_get_settings(index: @index_name).values.first
|
89
85
|
rescue Elasticsearch::Transport::Transport::Errors::NotFound
|
90
86
|
nil
|
91
87
|
end
|
92
88
|
|
93
89
|
def mappings
|
94
90
|
ActiveSupport::Deprecation.warn(
|
95
|
-
|
96
|
-
|
91
|
+
"Elasticity::Strategies::SingleIndex#mappings is deprecated, "\
|
92
|
+
"use mapping instead"
|
97
93
|
)
|
98
94
|
mapping
|
99
95
|
end
|
100
96
|
|
101
97
|
def mapping
|
102
|
-
@client.index_get_mapping(index: @index_name
|
98
|
+
@client.index_get_mapping(index: @index_name).values.first
|
103
99
|
rescue Elasticsearch::Transport::Transport::Errors::NotFound
|
104
100
|
nil
|
105
101
|
end
|
data/lib/elasticity/version.rb
CHANGED
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: es-elasticity
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 1.0.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Rodrigo Kochenburger
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2022-08-
|
11
|
+
date: 2022-08-10 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: bundler
|