zelastic 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.circleci/config.yml +56 -0
- data/.gitignore +11 -0
- data/.rspec +3 -0
- data/.rubocop.yml +21 -0
- data/Gemfile +5 -0
- data/Gemfile.lock +78 -0
- data/LICENSE.txt +21 -0
- data/README.md +107 -0
- data/Rakefile +8 -0
- data/bin/console +12 -0
- data/bin/setup +8 -0
- data/lib/zelastic.rb +6 -0
- data/lib/zelastic/config.rb +53 -0
- data/lib/zelastic/index_manager.rb +107 -0
- data/lib/zelastic/indexer.rb +105 -0
- data/lib/zelastic/version.rb +5 -0
- data/zelastic.gemspec +34 -0
- metadata +159 -0
checksums.yaml
ADDED
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
---
|
|
2
|
+
SHA1:
|
|
3
|
+
metadata.gz: 32d5e7e9052a7ebbb7c2cbd634e44b526e1404b6
|
|
4
|
+
data.tar.gz: 35e76a4d208277eb3f6860a6653b8893ea969aaa
|
|
5
|
+
SHA512:
|
|
6
|
+
metadata.gz: fc224397b8174c2779939103cecd6fa4151da5e14847961a46258e28aa079631136a89fb29474af53a5d8c6c7173ca5c90df467d133d819fa4174ee2f725698d
|
|
7
|
+
data.tar.gz: 9bd14709ce77ca55689209e480021864cd42f1333d3d4d635bd8f8f886e9b7cfbbb6809f1331e83144fd6c91fcde5cca316dfa9baccce32ad6937cde75058110
|
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
ruby: &ruby
|
|
2
|
+
image: carwow/ruby-ci:2.4.2
|
|
3
|
+
|
|
4
|
+
elasticsearch_container: &elasticsearch_container
|
|
5
|
+
image: carwow/elasticsearch-ci:5.5.1
|
|
6
|
+
|
|
7
|
+
version: 2
|
|
8
|
+
jobs:
|
|
9
|
+
bundle:
|
|
10
|
+
working_directory: ~/zelastic
|
|
11
|
+
docker:
|
|
12
|
+
- *ruby
|
|
13
|
+
steps:
|
|
14
|
+
- checkout
|
|
15
|
+
- restore_cache:
|
|
16
|
+
keys:
|
|
17
|
+
- bundle-{{ checksum "Gemfile.lock" }}
|
|
18
|
+
- bundle-
|
|
19
|
+
- run: |
|
|
20
|
+
bundle config --local path vendor/bundle &&
|
|
21
|
+
bundle check || bundle install --jobs=4 --retry=3
|
|
22
|
+
- save_cache:
|
|
23
|
+
key: bundle-{{ checksum "Gemfile.lock" }}
|
|
24
|
+
paths: [~/zelastic/vendor/bundle]
|
|
25
|
+
- persist_to_workspace:
|
|
26
|
+
root: '~'
|
|
27
|
+
paths: [zelastic]
|
|
28
|
+
|
|
29
|
+
rubocop:
|
|
30
|
+
working_directory: ~/zelastic
|
|
31
|
+
docker:
|
|
32
|
+
- *ruby
|
|
33
|
+
steps:
|
|
34
|
+
- attach_workspace:
|
|
35
|
+
at: '~'
|
|
36
|
+
- run: bundle exec rubocop
|
|
37
|
+
|
|
38
|
+
tests:
|
|
39
|
+
working_directory: ~/zelastic
|
|
40
|
+
docker:
|
|
41
|
+
- *ruby
|
|
42
|
+
- *elasticsearch_container
|
|
43
|
+
steps:
|
|
44
|
+
- attach_workspace:
|
|
45
|
+
at: '~'
|
|
46
|
+
- run: |
|
|
47
|
+
bundle exec rspec --pattern "**/*_spec.rb" --format "progress"
|
|
48
|
+
workflows:
|
|
49
|
+
version: 2
|
|
50
|
+
build:
|
|
51
|
+
jobs:
|
|
52
|
+
- bundle
|
|
53
|
+
- rubocop:
|
|
54
|
+
requires: [bundle]
|
|
55
|
+
- tests:
|
|
56
|
+
requires: [bundle]
|
data/.gitignore
ADDED
data/.rspec
ADDED
data/.rubocop.yml
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
AllCops:
|
|
2
|
+
TargetRubyVersion: 2.4
|
|
3
|
+
|
|
4
|
+
# Configuration parameters: CountComments, ExcludedMethods.
|
|
5
|
+
Metrics/BlockLength:
|
|
6
|
+
Exclude:
|
|
7
|
+
- '*.gemspec'
|
|
8
|
+
|
|
9
|
+
# Configuration parameters: CountComments.
|
|
10
|
+
Metrics/MethodLength:
|
|
11
|
+
Max: 18
|
|
12
|
+
|
|
13
|
+
# Offense count: 2
|
|
14
|
+
Style/Documentation:
|
|
15
|
+
Exclude:
|
|
16
|
+
- '**/*'
|
|
17
|
+
|
|
18
|
+
# Configuration parameters: AllowHeredoc, AllowURI, URISchemes, IgnoreCopDirectives, IgnoredPatterns.
|
|
19
|
+
# URISchemes: http, https
|
|
20
|
+
Metrics/LineLength:
|
|
21
|
+
Max: 102
|
data/Gemfile
ADDED
data/Gemfile.lock
ADDED
|
@@ -0,0 +1,78 @@
|
|
|
1
|
+
PATH
|
|
2
|
+
remote: .
|
|
3
|
+
specs:
|
|
4
|
+
zelastic (0.1.0)
|
|
5
|
+
activesupport
|
|
6
|
+
|
|
7
|
+
GEM
|
|
8
|
+
remote: https://rubygems.org/
|
|
9
|
+
specs:
|
|
10
|
+
activemodel (5.1.4)
|
|
11
|
+
activesupport (= 5.1.4)
|
|
12
|
+
activerecord (5.1.4)
|
|
13
|
+
activemodel (= 5.1.4)
|
|
14
|
+
activesupport (= 5.1.4)
|
|
15
|
+
arel (~> 8.0)
|
|
16
|
+
activesupport (5.1.4)
|
|
17
|
+
concurrent-ruby (~> 1.0, >= 1.0.2)
|
|
18
|
+
i18n (~> 0.7)
|
|
19
|
+
minitest (~> 5.1)
|
|
20
|
+
tzinfo (~> 1.1)
|
|
21
|
+
arel (8.0.0)
|
|
22
|
+
ast (2.3.0)
|
|
23
|
+
coderay (1.1.2)
|
|
24
|
+
concurrent-ruby (1.0.5)
|
|
25
|
+
diff-lcs (1.3)
|
|
26
|
+
i18n (0.9.1)
|
|
27
|
+
concurrent-ruby (~> 1.0)
|
|
28
|
+
method_source (0.9.0)
|
|
29
|
+
minitest (5.11.1)
|
|
30
|
+
parallel (1.12.1)
|
|
31
|
+
parser (2.4.0.2)
|
|
32
|
+
ast (~> 2.3)
|
|
33
|
+
powerpack (0.1.1)
|
|
34
|
+
pry (0.11.3)
|
|
35
|
+
coderay (~> 1.1.0)
|
|
36
|
+
method_source (~> 0.9.0)
|
|
37
|
+
rainbow (3.0.0)
|
|
38
|
+
rake (12.3.0)
|
|
39
|
+
rspec (3.7.0)
|
|
40
|
+
rspec-core (~> 3.7.0)
|
|
41
|
+
rspec-expectations (~> 3.7.0)
|
|
42
|
+
rspec-mocks (~> 3.7.0)
|
|
43
|
+
rspec-core (3.7.1)
|
|
44
|
+
rspec-support (~> 3.7.0)
|
|
45
|
+
rspec-expectations (3.7.0)
|
|
46
|
+
diff-lcs (>= 1.2.0, < 2.0)
|
|
47
|
+
rspec-support (~> 3.7.0)
|
|
48
|
+
rspec-mocks (3.7.0)
|
|
49
|
+
diff-lcs (>= 1.2.0, < 2.0)
|
|
50
|
+
rspec-support (~> 3.7.0)
|
|
51
|
+
rspec-support (3.7.0)
|
|
52
|
+
rubocop (0.52.1)
|
|
53
|
+
parallel (~> 1.10)
|
|
54
|
+
parser (>= 2.4.0.2, < 3.0)
|
|
55
|
+
powerpack (~> 0.1)
|
|
56
|
+
rainbow (>= 2.2.2, < 4.0)
|
|
57
|
+
ruby-progressbar (~> 1.7)
|
|
58
|
+
unicode-display_width (~> 1.0, >= 1.0.1)
|
|
59
|
+
ruby-progressbar (1.9.0)
|
|
60
|
+
thread_safe (0.3.6)
|
|
61
|
+
tzinfo (1.2.4)
|
|
62
|
+
thread_safe (~> 0.1)
|
|
63
|
+
unicode-display_width (1.3.0)
|
|
64
|
+
|
|
65
|
+
PLATFORMS
|
|
66
|
+
ruby
|
|
67
|
+
|
|
68
|
+
DEPENDENCIES
|
|
69
|
+
activerecord
|
|
70
|
+
bundler
|
|
71
|
+
pry
|
|
72
|
+
rake
|
|
73
|
+
rspec
|
|
74
|
+
rubocop
|
|
75
|
+
zelastic!
|
|
76
|
+
|
|
77
|
+
BUNDLED WITH
|
|
78
|
+
1.16.1
|
data/LICENSE.txt
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
The MIT License (MIT)
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2018 Isaac Seymour
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in
|
|
13
|
+
all copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
|
21
|
+
THE SOFTWARE.
|
data/README.md
ADDED
|
@@ -0,0 +1,107 @@
|
|
|
1
|
+
# Zero-downtime indexing from ActiveRecord->Elasticsearch
|
|
2
|
+
|
|
3
|
+
## Installation
|
|
4
|
+
|
|
5
|
+
Add this line to your application's Gemfile:
|
|
6
|
+
|
|
7
|
+
```ruby
|
|
8
|
+
gem 'zelastic'
|
|
9
|
+
```
|
|
10
|
+
|
|
11
|
+
And then execute:
|
|
12
|
+
|
|
13
|
+
$ bundle
|
|
14
|
+
|
|
15
|
+
Or install it yourself as:
|
|
16
|
+
|
|
17
|
+
$ gem install zelastic
|
|
18
|
+
|
|
19
|
+
## Usage
|
|
20
|
+
### Setup
|
|
21
|
+
|
|
22
|
+
For each ActiveRecord scope you want to index, you'll need a configuration:
|
|
23
|
+
```ruby
|
|
24
|
+
class MyModel < ApplicationRecord
|
|
25
|
+
...
|
|
26
|
+
end
|
|
27
|
+
|
|
28
|
+
MyModelIndex = Zelastic.new(
|
|
29
|
+
client: Elasticsearch::Client.new(...),
|
|
30
|
+
mapping: {
|
|
31
|
+
...
|
|
32
|
+
},
|
|
33
|
+
data_source: MyModel.some_scope
|
|
34
|
+
) do |my_model|
|
|
35
|
+
# this block transforms an instance of MyModel into the hash which goes into Elasticsearch
|
|
36
|
+
{
|
|
37
|
+
attr_1: my_model.attr_1,
|
|
38
|
+
attr_2: my_model.attr_2,
|
|
39
|
+
attr_3: my_model.attr_3
|
|
40
|
+
}
|
|
41
|
+
end
|
|
42
|
+
```
|
|
43
|
+
|
|
44
|
+
You can also override some defaults, if you wish:
|
|
45
|
+
- `index_settings`: by default there aren't any, but you can provide, for example, custom analysers
|
|
46
|
+
here
|
|
47
|
+
- `read_alias`: by default this is the table name of the `data_source`
|
|
48
|
+
- `write_alias`: by default this is the `read_alias`, with `_write` appended
|
|
49
|
+
- `type`: by default this is `read_alias.singularize`
|
|
50
|
+
|
|
51
|
+
### Normal usage
|
|
52
|
+
|
|
53
|
+
You'll need to make sure the following gets run whenever an instance of MyModel is updated:
|
|
54
|
+
|
|
55
|
+
```ruby
|
|
56
|
+
indexer = Zelastic::Indexer.new(MyModelIndex)
|
|
57
|
+
indexer.index_record(my_model)
|
|
58
|
+
```
|
|
59
|
+
|
|
60
|
+
And when an instance of MyModel gets deleted:
|
|
61
|
+
```ruby
|
|
62
|
+
indexer = Zelastic::Indexer.new(MyModelIndex)
|
|
63
|
+
indexer.delete_by_id(my_model.id)
|
|
64
|
+
```
|
|
65
|
+
|
|
66
|
+
There's also some bulk-change methods which may be useful:
|
|
67
|
+
```ruby
|
|
68
|
+
indexer = Zelastic::Indexer.new(MyModelIndex)
|
|
69
|
+
indexer.index_batch(MyModel.where(id: [...]))
|
|
70
|
+
indexer.delete_by_ids([1, 2, 3])
|
|
71
|
+
indexer.delete_by_query(elasticsearch_query)
|
|
72
|
+
```
|
|
73
|
+
|
|
74
|
+
### Re-indexing
|
|
75
|
+
|
|
76
|
+
Sometimes you'll need to do a full reindex - maybe because of a bug which left the index in a bad
|
|
77
|
+
state, or because of a new index definition, or...anything else.
|
|
78
|
+
|
|
79
|
+
We use index aliases to make it easy to do zero-downtime reindexing. The actual indexes are
|
|
80
|
+
`<read_alias>_<random>`. The `read_alias` points to the single "current" index.
|
|
81
|
+
The `write_alias` is usually the same as the read alias, except during re-indexing, where it
|
|
82
|
+
points at both the old and new indices, so both receive writes. The following steps run a
|
|
83
|
+
full reindex:
|
|
84
|
+
|
|
85
|
+
1. `new_name = SecureRandom.hex(3)`
|
|
86
|
+
2. `index_manager = Zelastic::IndexManager.new(MyModelIndex)`
|
|
87
|
+
2. `index_manager.create_index(new_name)`
|
|
88
|
+
3. `index_manager.populate_index(new_name, batch_size: 3000)`
|
|
89
|
+
4. Check that the new index is looking alrightish
|
|
90
|
+
5. `index_manager.switch_read_index(new_name)`
|
|
91
|
+
6. Probably do some more checks, then
|
|
92
|
+
7. `index_manager.stop_dual_writes`
|
|
93
|
+
8. `index_manager.cleanup_old_indices`
|
|
94
|
+
|
|
95
|
+
## Development
|
|
96
|
+
|
|
97
|
+
After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
|
|
98
|
+
|
|
99
|
+
To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
|
|
100
|
+
|
|
101
|
+
## Contributing
|
|
102
|
+
|
|
103
|
+
Bug reports and pull requests are welcome on GitHub at https://github.com/carwow/zelastic.
|
|
104
|
+
|
|
105
|
+
## License
|
|
106
|
+
|
|
107
|
+
The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT).
|
data/Rakefile
ADDED
data/bin/console
ADDED
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
#!/usr/bin/env ruby
|
|
2
|
+
# frozen_string_literal: true
|
|
3
|
+
|
|
4
|
+
require 'bundler/setup'
|
|
5
|
+
require 'zelastic'
|
|
6
|
+
|
|
7
|
+
# You can add fixtures and/or initialization code here to make experimenting
|
|
8
|
+
# with your gem easier. You can also use a different console, if you like.
|
|
9
|
+
|
|
10
|
+
# (If you use this, don't forget to add pry to your Gemfile!)
|
|
11
|
+
require 'pry'
|
|
12
|
+
Pry.start
|
data/bin/setup
ADDED
data/lib/zelastic.rb
ADDED
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Zelastic
|
|
4
|
+
class Config
|
|
5
|
+
attr_reader :client, :data_source
|
|
6
|
+
|
|
7
|
+
def initialize(
|
|
8
|
+
client:,
|
|
9
|
+
data_source:,
|
|
10
|
+
mapping:,
|
|
11
|
+
**overrides,
|
|
12
|
+
&index_data
|
|
13
|
+
)
|
|
14
|
+
@client = client
|
|
15
|
+
@data_source = data_source
|
|
16
|
+
@mapping = mapping
|
|
17
|
+
@index_data = index_data
|
|
18
|
+
@overrides = overrides
|
|
19
|
+
end
|
|
20
|
+
|
|
21
|
+
def index_data(model)
|
|
22
|
+
@index_data.call(model)
|
|
23
|
+
end
|
|
24
|
+
|
|
25
|
+
def read_alias
|
|
26
|
+
@read_alias ||= overrides.fetch(:read_alias) { data_source.table_name }
|
|
27
|
+
end
|
|
28
|
+
|
|
29
|
+
def write_alias
|
|
30
|
+
@write_alias ||= overrides.fetch(:write_alias) { [read_alias, 'write'].join('_') }
|
|
31
|
+
end
|
|
32
|
+
|
|
33
|
+
def type
|
|
34
|
+
@type ||= overrides.fetch(:type, read_alias.singularize)
|
|
35
|
+
end
|
|
36
|
+
|
|
37
|
+
def logger
|
|
38
|
+
return Rails.logger if defined?(Rails)
|
|
39
|
+
@logger ||= Logger.new(STDOUT)
|
|
40
|
+
end
|
|
41
|
+
|
|
42
|
+
def index_definition
|
|
43
|
+
{
|
|
44
|
+
settings: overrides.fetch(:index_settings, {}),
|
|
45
|
+
mappings: { type => mapping }
|
|
46
|
+
}
|
|
47
|
+
end
|
|
48
|
+
|
|
49
|
+
private
|
|
50
|
+
|
|
51
|
+
attr_reader :overrides, :mapping
|
|
52
|
+
end
|
|
53
|
+
end
|
|
@@ -0,0 +1,107 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Zelastic
|
|
4
|
+
class IndexManager
|
|
5
|
+
extend Forwardable
|
|
6
|
+
|
|
7
|
+
def initialize(config)
|
|
8
|
+
@config = config
|
|
9
|
+
end
|
|
10
|
+
|
|
11
|
+
def create_index(unique_name)
|
|
12
|
+
full_name = [config.read_alias, unique_name].join('_')
|
|
13
|
+
|
|
14
|
+
client.indices.create(
|
|
15
|
+
index: full_name,
|
|
16
|
+
body: config.index_definition
|
|
17
|
+
)
|
|
18
|
+
|
|
19
|
+
client.indices.put_alias(index: full_name, name: config.write_alias)
|
|
20
|
+
end
|
|
21
|
+
|
|
22
|
+
def populate_index(unique_name = nil, batch_size: 3000)
|
|
23
|
+
index_name = if unique_name
|
|
24
|
+
[config.read_alias, unique_name].join('_')
|
|
25
|
+
else
|
|
26
|
+
config.write_alias
|
|
27
|
+
end
|
|
28
|
+
|
|
29
|
+
config.data_source.find_in_batches(batch_size: batch_size) do |batch|
|
|
30
|
+
indexer.index_batch(batch, index_name: index_name)
|
|
31
|
+
end
|
|
32
|
+
end
|
|
33
|
+
|
|
34
|
+
# rubocop:disable Metrics/AbcSize
|
|
35
|
+
def switch_read_index(new_name)
|
|
36
|
+
new_index = [config.read_alias, new_name].join('_')
|
|
37
|
+
|
|
38
|
+
old_index =
|
|
39
|
+
if client.indices.exists_alias?(name: config.read_alias)
|
|
40
|
+
client.indices.get_alias(name: config.read_alias).keys.first
|
|
41
|
+
end
|
|
42
|
+
|
|
43
|
+
remove_action =
|
|
44
|
+
({ remove: { index: old_index, alias: config.read_alias } } if old_index)
|
|
45
|
+
|
|
46
|
+
client.indices.update_aliases(body: {
|
|
47
|
+
actions: [
|
|
48
|
+
remove_action,
|
|
49
|
+
{ add: { index: new_index, alias: config.read_alias } }
|
|
50
|
+
].compact
|
|
51
|
+
})
|
|
52
|
+
end
|
|
53
|
+
|
|
54
|
+
def stop_dual_writes
|
|
55
|
+
logger.info('Stopping dual writes - making index read and write aliases the same')
|
|
56
|
+
current_index = client.indices.get_alias(name: config.read_alias).keys.first
|
|
57
|
+
|
|
58
|
+
logger.info("Currently used index is #{current_index}")
|
|
59
|
+
|
|
60
|
+
other_write_indices = client.indices.get_alias(name: config.write_alias).keys
|
|
61
|
+
.reject { |name| name == current_index }
|
|
62
|
+
|
|
63
|
+
if other_write_indices.none?
|
|
64
|
+
logger.info("No write indexes that aren't the read index. Nothing to do!")
|
|
65
|
+
return
|
|
66
|
+
end
|
|
67
|
+
logger.info("Stopping writes to #{other_write_indices.count} old ES indices: " \
|
|
68
|
+
"#{other_write_indices.join(', ')}")
|
|
69
|
+
|
|
70
|
+
actions = other_write_indices.map do |index|
|
|
71
|
+
{ remove: { index: index, alias: config.write_alias } }
|
|
72
|
+
end
|
|
73
|
+
client.indices.update_aliases(body: { actions: actions })
|
|
74
|
+
end
|
|
75
|
+
|
|
76
|
+
def cleanup_old_indices
|
|
77
|
+
logger.info('Cleaning up old indices in Elasticsearch')
|
|
78
|
+
current_index = client.indices.get_alias(name: config.read_alias).keys.first
|
|
79
|
+
|
|
80
|
+
logger.info("Currently used index is #{current_index}")
|
|
81
|
+
|
|
82
|
+
indices_to_delete = client
|
|
83
|
+
.cat
|
|
84
|
+
.indices(format: :json)
|
|
85
|
+
.map { |index| index['index'] }
|
|
86
|
+
.select { |name| name.start_with?(config.read_alias) }
|
|
87
|
+
.reject { |name| name == current_index }
|
|
88
|
+
|
|
89
|
+
if indices_to_delete.none?
|
|
90
|
+
logger.info('Nothing to do: no old indices')
|
|
91
|
+
return
|
|
92
|
+
end
|
|
93
|
+
logger.info("Deleting #{indices_to_delete.count} old indices: #{indices_to_delete.join(', ')}")
|
|
94
|
+
client.indices.delete(index: indices_to_delete)
|
|
95
|
+
end
|
|
96
|
+
# rubocop:enable Metrics/AbcSize
|
|
97
|
+
|
|
98
|
+
private
|
|
99
|
+
|
|
100
|
+
attr_reader :config
|
|
101
|
+
def_delegators :config, :client, :logger
|
|
102
|
+
|
|
103
|
+
def indexer
|
|
104
|
+
@indexer ||= Indexer.new(config)
|
|
105
|
+
end
|
|
106
|
+
end
|
|
107
|
+
end
|
|
@@ -0,0 +1,105 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Zelastic
|
|
4
|
+
class Indexer
|
|
5
|
+
extend Forwardable
|
|
6
|
+
|
|
7
|
+
def initialize(config)
|
|
8
|
+
@config = config
|
|
9
|
+
end
|
|
10
|
+
|
|
11
|
+
def index_batch(batch, index_name: nil)
|
|
12
|
+
indices = Array(index_name || write_indices)
|
|
13
|
+
logger.info("ES: Indexing #{config.type} record")
|
|
14
|
+
|
|
15
|
+
version = current_version
|
|
16
|
+
execute_bulk(
|
|
17
|
+
indices.flat_map do |index|
|
|
18
|
+
batch.map do |record|
|
|
19
|
+
index_command(index: index, version: version, record: record)
|
|
20
|
+
end
|
|
21
|
+
end
|
|
22
|
+
)
|
|
23
|
+
end
|
|
24
|
+
|
|
25
|
+
def index_record(record)
|
|
26
|
+
version = current_version
|
|
27
|
+
|
|
28
|
+
execute_bulk(
|
|
29
|
+
write_indices.map do |index|
|
|
30
|
+
index_command(index: index, version: version, record: record)
|
|
31
|
+
end
|
|
32
|
+
)
|
|
33
|
+
end
|
|
34
|
+
|
|
35
|
+
def delete_by_id(id)
|
|
36
|
+
indices = client.indices.get_alias(name: config.write_alias).keys
|
|
37
|
+
|
|
38
|
+
indices.each do |index|
|
|
39
|
+
client.delete(
|
|
40
|
+
index: index,
|
|
41
|
+
type: config.type,
|
|
42
|
+
id: id
|
|
43
|
+
)
|
|
44
|
+
end
|
|
45
|
+
end
|
|
46
|
+
|
|
47
|
+
def delete_by_ids(ids)
|
|
48
|
+
logger.info('ES: Deleting batch records')
|
|
49
|
+
|
|
50
|
+
indices = config.client.indices.get_alias(name: config.write_alias).keys
|
|
51
|
+
|
|
52
|
+
execute_bulk(
|
|
53
|
+
indices.flat_map do |index|
|
|
54
|
+
ids.map do |id|
|
|
55
|
+
{
|
|
56
|
+
delete: {
|
|
57
|
+
_index: index,
|
|
58
|
+
_type: config.type,
|
|
59
|
+
_id: id
|
|
60
|
+
}
|
|
61
|
+
}
|
|
62
|
+
end
|
|
63
|
+
end
|
|
64
|
+
)
|
|
65
|
+
end
|
|
66
|
+
|
|
67
|
+
def delete_by_query(query)
|
|
68
|
+
logger.info('ES: Deleting batch records')
|
|
69
|
+
|
|
70
|
+
config.client.delete_by_query(index: config.write_alias, body: { query: query })
|
|
71
|
+
end
|
|
72
|
+
|
|
73
|
+
private
|
|
74
|
+
|
|
75
|
+
attr_reader :config
|
|
76
|
+
def_delegators :config, :logger
|
|
77
|
+
|
|
78
|
+
def current_version
|
|
79
|
+
config.data_source.connection.select_one('SELECT txid_current()').fetch('txid_current')
|
|
80
|
+
end
|
|
81
|
+
|
|
82
|
+
def write_indices
|
|
83
|
+
config.client.indices.get_alias(name: config.write_alias).keys
|
|
84
|
+
end
|
|
85
|
+
|
|
86
|
+
def index_command(index:, version:, record:)
|
|
87
|
+
{
|
|
88
|
+
index: {
|
|
89
|
+
_index: index,
|
|
90
|
+
_type: config.type,
|
|
91
|
+
_id: record.id,
|
|
92
|
+
_version: version,
|
|
93
|
+
_version_type: :external,
|
|
94
|
+
data: config.index_data(record)
|
|
95
|
+
}
|
|
96
|
+
}
|
|
97
|
+
end
|
|
98
|
+
|
|
99
|
+
def execute_bulk(commands)
|
|
100
|
+
result = config.client.bulk(body: commands)
|
|
101
|
+
return result unless result['errors']
|
|
102
|
+
result['items'].map { |item| item['error'] }.compact
|
|
103
|
+
end
|
|
104
|
+
end
|
|
105
|
+
end
|
data/zelastic.gemspec
ADDED
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
|
|
2
|
+
# frozen_string_literal: true
|
|
3
|
+
|
|
4
|
+
lib = File.expand_path('../lib', __FILE__)
|
|
5
|
+
$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
|
6
|
+
require 'zelastic/version'
|
|
7
|
+
|
|
8
|
+
Gem::Specification.new do |spec|
|
|
9
|
+
spec.name = 'zelastic'
|
|
10
|
+
spec.version = Zelastic::VERSION
|
|
11
|
+
spec.authors = ['carwow Developers']
|
|
12
|
+
spec.email = ['developers@carwow.co.uk']
|
|
13
|
+
|
|
14
|
+
spec.summary = 'Zero-downtime (re-)indexing of ActiveRecord models into Elasticsearch.'
|
|
15
|
+
spec.description = 'An index manager for Elasticsearch and ActiveRecord'
|
|
16
|
+
spec.homepage = 'https://github.com/carwow/zelastic'
|
|
17
|
+
spec.license = 'MIT'
|
|
18
|
+
|
|
19
|
+
spec.files = `git ls-files -z`.split("\x0").reject do |f|
|
|
20
|
+
f.match(%r{^(test|spec|features)/})
|
|
21
|
+
end
|
|
22
|
+
spec.bindir = 'exe'
|
|
23
|
+
spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
|
|
24
|
+
spec.require_paths = ['lib']
|
|
25
|
+
|
|
26
|
+
spec.add_dependency 'activesupport'
|
|
27
|
+
|
|
28
|
+
spec.add_development_dependency 'activerecord'
|
|
29
|
+
spec.add_development_dependency 'bundler'
|
|
30
|
+
spec.add_development_dependency 'pry'
|
|
31
|
+
spec.add_development_dependency 'rake'
|
|
32
|
+
spec.add_development_dependency 'rspec'
|
|
33
|
+
spec.add_development_dependency 'rubocop'
|
|
34
|
+
end
|
metadata
ADDED
|
@@ -0,0 +1,159 @@
|
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
|
2
|
+
name: zelastic
|
|
3
|
+
version: !ruby/object:Gem::Version
|
|
4
|
+
version: 0.1.0
|
|
5
|
+
platform: ruby
|
|
6
|
+
authors:
|
|
7
|
+
- carwow Developers
|
|
8
|
+
autorequire:
|
|
9
|
+
bindir: exe
|
|
10
|
+
cert_chain: []
|
|
11
|
+
date: 2018-01-17 00:00:00.000000000 Z
|
|
12
|
+
dependencies:
|
|
13
|
+
- !ruby/object:Gem::Dependency
|
|
14
|
+
name: activesupport
|
|
15
|
+
requirement: !ruby/object:Gem::Requirement
|
|
16
|
+
requirements:
|
|
17
|
+
- - ">="
|
|
18
|
+
- !ruby/object:Gem::Version
|
|
19
|
+
version: '0'
|
|
20
|
+
type: :runtime
|
|
21
|
+
prerelease: false
|
|
22
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
23
|
+
requirements:
|
|
24
|
+
- - ">="
|
|
25
|
+
- !ruby/object:Gem::Version
|
|
26
|
+
version: '0'
|
|
27
|
+
- !ruby/object:Gem::Dependency
|
|
28
|
+
name: activerecord
|
|
29
|
+
requirement: !ruby/object:Gem::Requirement
|
|
30
|
+
requirements:
|
|
31
|
+
- - ">="
|
|
32
|
+
- !ruby/object:Gem::Version
|
|
33
|
+
version: '0'
|
|
34
|
+
type: :development
|
|
35
|
+
prerelease: false
|
|
36
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
37
|
+
requirements:
|
|
38
|
+
- - ">="
|
|
39
|
+
- !ruby/object:Gem::Version
|
|
40
|
+
version: '0'
|
|
41
|
+
- !ruby/object:Gem::Dependency
|
|
42
|
+
name: bundler
|
|
43
|
+
requirement: !ruby/object:Gem::Requirement
|
|
44
|
+
requirements:
|
|
45
|
+
- - ">="
|
|
46
|
+
- !ruby/object:Gem::Version
|
|
47
|
+
version: '0'
|
|
48
|
+
type: :development
|
|
49
|
+
prerelease: false
|
|
50
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
51
|
+
requirements:
|
|
52
|
+
- - ">="
|
|
53
|
+
- !ruby/object:Gem::Version
|
|
54
|
+
version: '0'
|
|
55
|
+
- !ruby/object:Gem::Dependency
|
|
56
|
+
name: pry
|
|
57
|
+
requirement: !ruby/object:Gem::Requirement
|
|
58
|
+
requirements:
|
|
59
|
+
- - ">="
|
|
60
|
+
- !ruby/object:Gem::Version
|
|
61
|
+
version: '0'
|
|
62
|
+
type: :development
|
|
63
|
+
prerelease: false
|
|
64
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
65
|
+
requirements:
|
|
66
|
+
- - ">="
|
|
67
|
+
- !ruby/object:Gem::Version
|
|
68
|
+
version: '0'
|
|
69
|
+
- !ruby/object:Gem::Dependency
|
|
70
|
+
name: rake
|
|
71
|
+
requirement: !ruby/object:Gem::Requirement
|
|
72
|
+
requirements:
|
|
73
|
+
- - ">="
|
|
74
|
+
- !ruby/object:Gem::Version
|
|
75
|
+
version: '0'
|
|
76
|
+
type: :development
|
|
77
|
+
prerelease: false
|
|
78
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
79
|
+
requirements:
|
|
80
|
+
- - ">="
|
|
81
|
+
- !ruby/object:Gem::Version
|
|
82
|
+
version: '0'
|
|
83
|
+
- !ruby/object:Gem::Dependency
|
|
84
|
+
name: rspec
|
|
85
|
+
requirement: !ruby/object:Gem::Requirement
|
|
86
|
+
requirements:
|
|
87
|
+
- - ">="
|
|
88
|
+
- !ruby/object:Gem::Version
|
|
89
|
+
version: '0'
|
|
90
|
+
type: :development
|
|
91
|
+
prerelease: false
|
|
92
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
93
|
+
requirements:
|
|
94
|
+
- - ">="
|
|
95
|
+
- !ruby/object:Gem::Version
|
|
96
|
+
version: '0'
|
|
97
|
+
- !ruby/object:Gem::Dependency
|
|
98
|
+
name: rubocop
|
|
99
|
+
requirement: !ruby/object:Gem::Requirement
|
|
100
|
+
requirements:
|
|
101
|
+
- - ">="
|
|
102
|
+
- !ruby/object:Gem::Version
|
|
103
|
+
version: '0'
|
|
104
|
+
type: :development
|
|
105
|
+
prerelease: false
|
|
106
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
107
|
+
requirements:
|
|
108
|
+
- - ">="
|
|
109
|
+
- !ruby/object:Gem::Version
|
|
110
|
+
version: '0'
|
|
111
|
+
description: An index manager for Elasticsearch and ActiveRecord
|
|
112
|
+
email:
|
|
113
|
+
- developers@carwow.co.uk
|
|
114
|
+
executables: []
|
|
115
|
+
extensions: []
|
|
116
|
+
extra_rdoc_files: []
|
|
117
|
+
files:
|
|
118
|
+
- ".circleci/config.yml"
|
|
119
|
+
- ".gitignore"
|
|
120
|
+
- ".rspec"
|
|
121
|
+
- ".rubocop.yml"
|
|
122
|
+
- Gemfile
|
|
123
|
+
- Gemfile.lock
|
|
124
|
+
- LICENSE.txt
|
|
125
|
+
- README.md
|
|
126
|
+
- Rakefile
|
|
127
|
+
- bin/console
|
|
128
|
+
- bin/setup
|
|
129
|
+
- lib/zelastic.rb
|
|
130
|
+
- lib/zelastic/config.rb
|
|
131
|
+
- lib/zelastic/index_manager.rb
|
|
132
|
+
- lib/zelastic/indexer.rb
|
|
133
|
+
- lib/zelastic/version.rb
|
|
134
|
+
- zelastic.gemspec
|
|
135
|
+
homepage: https://github.com/carwow/zelastic
|
|
136
|
+
licenses:
|
|
137
|
+
- MIT
|
|
138
|
+
metadata: {}
|
|
139
|
+
post_install_message:
|
|
140
|
+
rdoc_options: []
|
|
141
|
+
require_paths:
|
|
142
|
+
- lib
|
|
143
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
|
144
|
+
requirements:
|
|
145
|
+
- - ">="
|
|
146
|
+
- !ruby/object:Gem::Version
|
|
147
|
+
version: '0'
|
|
148
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
|
149
|
+
requirements:
|
|
150
|
+
- - ">="
|
|
151
|
+
- !ruby/object:Gem::Version
|
|
152
|
+
version: '0'
|
|
153
|
+
requirements: []
|
|
154
|
+
rubyforge_project:
|
|
155
|
+
rubygems_version: 2.6.13
|
|
156
|
+
signing_key:
|
|
157
|
+
specification_version: 4
|
|
158
|
+
summary: Zero-downtime (re-)indexing of ActiveRecord models into Elasticsearch.
|
|
159
|
+
test_files: []
|