elasticsearch_record 1.5.3 → 1.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +47 -11
- data/docs/CHANGELOG.md +24 -2
- data/lib/active_record/connection_adapters/elasticsearch/table_statements.rb +62 -4
- data/lib/active_record/connection_adapters/elasticsearch/transactions.rb +54 -0
- data/lib/active_record/connection_adapters/elasticsearch/unsupported_implementation.rb +0 -7
- data/lib/active_record/connection_adapters/elasticsearch_adapter.rb +17 -3
- data/lib/arel/collectors/elasticsearch_query.rb +3 -0
- data/lib/elasticsearch_record/core.rb +15 -2
- data/lib/elasticsearch_record/gem_version.rb +2 -2
- data/lib/elasticsearch_record/instrumentation/log_subscriber.rb +7 -3
- data/lib/elasticsearch_record/model_api.rb +68 -10
- data/lib/elasticsearch_record/model_schema.rb +2 -5
- data/lib/elasticsearch_record/persistence.rb +13 -1
- data/lib/elasticsearch_record/query.rb +8 -4
- data/lib/elasticsearch_record/relation/core_methods.rb +12 -0
- data/lib/elasticsearch_record/relation/query_methods.rb +10 -0
- data/lib/elasticsearch_record/relation/result_methods.rb +42 -5
- data/lib/elasticsearch_record/result.rb +1 -1
- data/lib/elasticsearch_record.rb +10 -0
- metadata +3 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: b29c6db7894f8365eb5a5922633a4adf53a793006c3ffa2fb381dd9ae171b124
|
4
|
+
data.tar.gz: 72c7f4260b76be5743e061838df5051006c82a883172d29f2e0442b6215ac4ed
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 7e09f30a077c81524c800a5694947d947da7a0a9273a4304592221d2c191c8a8582fbd94e65990ddf21cdbb23d74d5e7d4e47e53e14139c362fda91468aae182
|
7
|
+
data.tar.gz: bb03f9c4fa2749ccd23be70de65a1480b2d0a7cfa89ef51f9a3c3c2cd7e0810708e9e1e8050d7de033bfc670d5a127c75349f1f1380578246107395746e41652
|
data/README.md
CHANGED
@@ -53,6 +53,22 @@ Or install it yourself as:
|
|
53
53
|
* logs Elasticsearch API-calls
|
54
54
|
* shows Runtime in logs
|
55
55
|
|
56
|
+
## Notice
|
57
|
+
Since ActiveRecord does not have any configuration option to support transactions and
|
58
|
+
Elasticsearch does **NOT** support transactions, it may be risky to ignore them.
|
59
|
+
|
60
|
+
As a default, transactions are 'silently swallowed' to not break any existing applications...
|
61
|
+
|
62
|
+
To raise an exception while using transactions on a ElasticsearchRecord model, the following flag can be enabled.
|
63
|
+
However enabling this flag will surely fail transactional tests _(prevent this with 'use_transactional_tests=false')_
|
64
|
+
|
65
|
+
```ruby
|
66
|
+
# config/initializers/elasticsearch_record.yml
|
67
|
+
|
68
|
+
# enable transactional exceptions
|
69
|
+
ElasticsearchRecord.error_on_transaction = true
|
70
|
+
```
|
71
|
+
|
56
72
|
## Setup
|
57
73
|
|
58
74
|
### a) Update your **database.yml** and add a elasticsearch connection:
|
@@ -224,6 +240,7 @@ total = scope.total
|
|
224
240
|
- configure
|
225
241
|
- aggregate
|
226
242
|
- refresh
|
243
|
+
- timeout
|
227
244
|
- query
|
228
245
|
- filter
|
229
246
|
- must_not
|
@@ -260,6 +277,7 @@ _see simple documentation about these methods @ [rubydoc](https://rubydoc.info/g
|
|
260
277
|
- composite
|
261
278
|
- point_in_time
|
262
279
|
- pit_results
|
280
|
+
- pit_delete
|
263
281
|
|
264
282
|
_see simple documentation about these methods @ [rubydoc](https://rubydoc.info/gems/elasticsearch_record/ElasticsearchRecord/Relation/ResultMethods)_
|
265
283
|
|
@@ -366,13 +384,26 @@ SearchUser.api.mappings
|
|
366
384
|
SearchUser.api.insert([{name: 'Hans', age: 34}, {name: 'Peter', age: 22}])
|
367
385
|
```
|
368
386
|
|
387
|
+
### dangerous methods
|
369
388
|
* open!
|
370
389
|
* close!
|
371
390
|
* refresh!
|
372
391
|
* block!
|
373
392
|
* unblock!
|
393
|
+
|
394
|
+
### dangerous methods with args
|
395
|
+
* create!(...)
|
396
|
+
* clone!(...)
|
397
|
+
* rename!(...)
|
398
|
+
* backup!(...)
|
399
|
+
* restore!(...)
|
400
|
+
* reindex!(...)
|
401
|
+
|
402
|
+
### dangerous methods with confirm parameter
|
374
403
|
* drop!(confirm: true)
|
375
404
|
* truncate!(confirm: true)
|
405
|
+
|
406
|
+
### table methods
|
376
407
|
* mappings
|
377
408
|
* metas
|
378
409
|
* settings
|
@@ -380,17 +411,19 @@ SearchUser.api.insert([{name: 'Hans', age: 34}, {name: 'Peter', age: 22}])
|
|
380
411
|
* state
|
381
412
|
* schema
|
382
413
|
* exists?
|
383
|
-
|
384
|
-
|
385
|
-
*
|
386
|
-
*
|
387
|
-
|
388
|
-
|
389
|
-
|
390
|
-
|
391
|
-
*
|
392
|
-
*
|
393
|
-
*
|
414
|
+
|
415
|
+
### plain methods
|
416
|
+
* alias_exists?(...)
|
417
|
+
* setting_exists?(...)
|
418
|
+
* mapping_exists?(...)
|
419
|
+
* meta_exists?(...)
|
420
|
+
|
421
|
+
### Fast insert, update, delete raw data
|
422
|
+
* index(...)
|
423
|
+
* insert(...)
|
424
|
+
* update(...)
|
425
|
+
* delete(...)
|
426
|
+
* bulk(...)
|
394
427
|
|
395
428
|
-----
|
396
429
|
|
@@ -436,6 +469,9 @@ Access these methods through the model's connection or within any `Migration`.
|
|
436
469
|
- create_table
|
437
470
|
- change_table
|
438
471
|
- rename_table
|
472
|
+
- reindex_table
|
473
|
+
- backup_table
|
474
|
+
- restore_table
|
439
475
|
|
440
476
|
### table actions:
|
441
477
|
- change_meta
|
data/docs/CHANGELOG.md
CHANGED
@@ -1,5 +1,27 @@
|
|
1
1
|
# ElasticsearchRecord - CHANGELOG
|
2
2
|
|
3
|
+
## [1.6.0] - 2023-08-11
|
4
|
+
* [add] `ElasticsearchRecord::Base#undelegate_id_attribute_with` method to support a temporary 'undelegation' (used to create a new record)
|
5
|
+
* [add] `ElasticsearchRecord::Relation#timeout` to directly provide the timeout-parameter to the query
|
6
|
+
* [add] `ElasticsearchRecord.error_on_transaction`-flag to throw transactional errors (default: `false`) - this will now **IGNORE** all transactions
|
7
|
+
* [add] `ElasticsearchRecord::ModelApi` create!, clone!, rename!, backup!, restore! & reindex!-methods
|
8
|
+
* [add] `ElasticsearchRecord::Relation#pit_delete` which executes a delete query in a 'point_in_time' scope.
|
9
|
+
* [add] `ActiveRecord::ConnectionAdapters::Elasticsearch::TableStatements#backup_table` to create a backup (snapshot) of the entire table (index)
|
10
|
+
* [add] `ActiveRecord::ConnectionAdapters::Elasticsearch::TableStatements#restore_table` to restore a entire table (index)
|
11
|
+
* [add] `ActiveRecord::ConnectionAdapters::Elasticsearch::TableStatements#reindex_table` to copy documents from source to destination
|
12
|
+
* [ref] `ElasticsearchRecord::Base.delegate_id_attribute` now supports instance writer
|
13
|
+
* [ref] `ElasticsearchRecord::Relation#pit_results` adds `ids_only`-parameter to now support a simple return of the records-ids...
|
14
|
+
* [fix] Relation `#last`-method will raise an transport exception if cluster setting '**indices.id_field_data.enabled**' is disabled (now checks for `access_id_fielddata?`)
|
15
|
+
* [fix] ElasticsearchRecord-connection settings does not support `username` key
|
16
|
+
* [fix] ElasticsearchRecord-connection settings does not support `port` key
|
17
|
+
* [fix] `_id`-Attribute is erroneously defined as 'virtual' attribute - but is required for insert statements.
|
18
|
+
* [fix] unsupported **SAVEPOINT** transactions throws exceptions _(especially in tests)_
|
19
|
+
* [fix] `ElasticsearchRecord::ModelApi#bulk` does not recognize `'_id' / :_id` attribute
|
20
|
+
* [fix] `ElasticsearchRecord::ModelApi#bulk` does not correctly build the data-hash for `update`-operation _(missing 'doc'-node)_
|
21
|
+
* [ref] simplify `ElasticsearchRecord::Base#searchable_column_names`
|
22
|
+
* [fix] creating a new record does not recognize a manually provided `_id`-attribute
|
23
|
+
* [fix] creating a new record with active `delegate_id_attribute`-flag does not update the records `_id`.
|
24
|
+
|
3
25
|
## [1.5.3] - 2023-07-14
|
4
26
|
* [fix] `ElasticsearchRecord::Relation#where!` on nested, provided `:none` key
|
5
27
|
* [ref] minor code tweaks and comment updates
|
@@ -13,10 +35,10 @@
|
|
13
35
|
|
14
36
|
## [1.5.0] - 2023-07-10
|
15
37
|
* [add] additional `ElasticsearchRecord::ModelApi` methods **drop!** & **truncate!**, which have to be called with a `confirm:true` parameter
|
16
|
-
* [add]
|
38
|
+
* [add] `ElasticsearchRecord::Base.delegate_query_nil_limit` to automatically delegate a relations `limit(nil)`-call to the **max_result_window** _(set to 10.000 as default)_
|
17
39
|
* [add] `ActiveRecord::ConnectionAdapters::Elasticsearch::SchemaStatements#access_shard_doc?` which checks, if the **PIT**-shard_doc order is available
|
18
40
|
* [add] support for **_shard_doc** as a default order for `ElasticsearchRecord::Relation#pit_results`
|
19
|
-
* [ref]
|
41
|
+
* [ref] `ElasticsearchRecord::Base.relay_id_attribute` to a more coherent name: `delegate_id_attribute`
|
20
42
|
* [ref] `ElasticsearchRecord::Relation#ordered_relation` to optimize already ordered relations
|
21
43
|
* [ref] gemspecs to support different versions of Elasticsearch
|
22
44
|
* [ref] improved README
|
@@ -116,7 +116,7 @@ module ActiveRecord
|
|
116
116
|
# Defaults to false.
|
117
117
|
# @param [String] table_name
|
118
118
|
# @param [Boolean] if_exists
|
119
|
-
# @return [
|
119
|
+
# @return [Boolean] acknowledged status
|
120
120
|
def drop_table(table_name, if_exists: false, **)
|
121
121
|
schema_cache.clear_data_source_cache!(table_name)
|
122
122
|
api(:indices, :delete, { index: table_name, ignore: (if_exists ? 404 : nil) }, 'DROP TABLE').dig('acknowledged')
|
@@ -148,11 +148,12 @@ module ActiveRecord
|
|
148
148
|
end
|
149
149
|
end
|
150
150
|
|
151
|
-
# clones an entire table (index) to the provided +target_name+.
|
151
|
+
# clones an entire table (index) with its docs to the provided +target_name+.
|
152
152
|
# During cloning, the table will be automatically 'write'-blocked.
|
153
153
|
# @param [String] table_name
|
154
154
|
# @param [String] target_name
|
155
155
|
# @param [Hash] options
|
156
|
+
# @return [Boolean] acknowledged status
|
156
157
|
def clone_table(table_name, target_name, **options)
|
157
158
|
# create new definition
|
158
159
|
definition = clone_table_definition(table_name, target_name, **extract_table_options!(options))
|
@@ -168,6 +169,54 @@ module ActiveRecord
|
|
168
169
|
definition.exec!
|
169
170
|
end
|
170
171
|
|
172
|
+
# creates a backup (snapshot) of the entire table (index) from provided +table_name+.
|
173
|
+
# The backup will be closed, to prevent read/write access.
|
174
|
+
# The +target_name+ will be auto-generated, if not provided.
|
175
|
+
#
|
176
|
+
# @example
|
177
|
+
# backup_table('screenshots', to: 'screenshots-backup-v1')
|
178
|
+
#
|
179
|
+
# @param [String] table_name
|
180
|
+
# @param [String] to - target_name
|
181
|
+
# @param [Boolean] close - closes backup after creation (default: true)
|
182
|
+
# @return [String] backup_name
|
183
|
+
def backup_table(table_name, to: nil, close: true)
|
184
|
+
to ||= "#{table_name}-snapshot-#{Time.now.strftime('%s%3N')}"
|
185
|
+
raise ArgumentError, "unable to backup '#{table_name}' to already existing target '#{to}'!" if table_exists?(to)
|
186
|
+
|
187
|
+
clone_table(table_name, to)
|
188
|
+
close_table(to) if close
|
189
|
+
|
190
|
+
to
|
191
|
+
end
|
192
|
+
|
193
|
+
# restores a entire table (index) from provided +target_name+.
|
194
|
+
# The +table_name+ will be dropped, if exists.
|
195
|
+
# The +from+ will persist, if not provided +drop_backup:true+.
|
196
|
+
#
|
197
|
+
# @example
|
198
|
+
# restore_table('screenshots', from: 'screenshots-backup-v1')
|
199
|
+
#
|
200
|
+
# @param [String] table_name
|
201
|
+
# @param [String] from
|
202
|
+
# @param [String (frozen)] timeout - renaming timout (default: '30s')
|
203
|
+
# @param [Boolean] open - opens restored backup after creation (default: true)
|
204
|
+
# @return [Boolean] acknowledged status
|
205
|
+
def restore_table(table_name, from:, timeout: nil, open: true, drop_backup: false)
|
206
|
+
raise ArgumentError, "unable to restore from missing target '#{from}'!" unless table_exists?(from)
|
207
|
+
drop_table(table_name, if_exists: true)
|
208
|
+
|
209
|
+
# choose best strategy
|
210
|
+
if drop_backup
|
211
|
+
rename_table(from, table_name, timeout: timeout)
|
212
|
+
else
|
213
|
+
clone_table(from, table_name)
|
214
|
+
end
|
215
|
+
|
216
|
+
# open, if provided
|
217
|
+
open_table(from) if open
|
218
|
+
end
|
219
|
+
|
171
220
|
# renames a table (index) by executing multiple steps:
|
172
221
|
# - clone table
|
173
222
|
# - wait for 'green' state
|
@@ -178,11 +227,11 @@ module ActiveRecord
|
|
178
227
|
# @param [String] target_name
|
179
228
|
# @param [String (frozen)] timeout (default: '30s')
|
180
229
|
# @param [Hash] options - additional 'clone' options (like settings, alias, ...)
|
181
|
-
def rename_table(table_name, target_name, timeout:
|
230
|
+
def rename_table(table_name, target_name, timeout: nil, **options)
|
182
231
|
schema_cache.clear_data_source_cache!(table_name)
|
183
232
|
|
184
233
|
clone_table(table_name, target_name, **options)
|
185
|
-
cluster_health(index: target_name, wait_for_status: 'green', timeout: timeout)
|
234
|
+
cluster_health(index: target_name, wait_for_status: 'green', timeout: timeout.presence || '30s')
|
186
235
|
drop_table(table_name)
|
187
236
|
end
|
188
237
|
|
@@ -255,6 +304,15 @@ module ActiveRecord
|
|
255
304
|
definition.exec!
|
256
305
|
end
|
257
306
|
|
307
|
+
# Copies documents from a source to a destination.
|
308
|
+
# @param [String] table_name
|
309
|
+
# @param [String] target_name
|
310
|
+
# @param [Hash] options
|
311
|
+
# @return [Hash] reindex stats
|
312
|
+
def reindex_table(table_name, target_name, **options)
|
313
|
+
api(:core, :reindex, { body: { source: { index: table_name }, dest: { index: target_name } } }.merge(options), 'REINDEX TABLE')
|
314
|
+
end
|
315
|
+
|
258
316
|
# -- mapping -------------------------------------------------------------------------------------------------
|
259
317
|
|
260
318
|
def add_mapping(table_name, name, type, **options, &block)
|
@@ -0,0 +1,54 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module ActiveRecord
|
4
|
+
module ConnectionAdapters
|
5
|
+
module Elasticsearch
|
6
|
+
module Transactions
|
7
|
+
extend ActiveSupport::Concern
|
8
|
+
|
9
|
+
def transaction(*)
|
10
|
+
# since ActiveRecord does not have any configuration option to support transactions,
|
11
|
+
# this will be always false
|
12
|
+
# return super if supports_transactions?
|
13
|
+
#
|
14
|
+
# So, transactions are silently swallowed...
|
15
|
+
yield
|
16
|
+
end
|
17
|
+
|
18
|
+
# Begins the transaction (and turns off auto-committing).
|
19
|
+
def begin_db_transaction(*)
|
20
|
+
_throw_transaction_exception!(:begin_db_transaction)
|
21
|
+
end
|
22
|
+
|
23
|
+
# Commits the transaction (and turns on auto-committing).
|
24
|
+
def commit_db_transaction(*)
|
25
|
+
_throw_transaction_exception!(:commit_db_transaction)
|
26
|
+
end
|
27
|
+
|
28
|
+
# rollback transaction
|
29
|
+
def exec_rollback_db_transaction(*)
|
30
|
+
_throw_transaction_exception!(:exec_rollback_db_transaction)
|
31
|
+
end
|
32
|
+
|
33
|
+
def create_savepoint(*)
|
34
|
+
_throw_transaction_exception!(:create_savepoint)
|
35
|
+
end
|
36
|
+
|
37
|
+
def exec_rollback_to_savepoint(*)
|
38
|
+
_throw_transaction_exception!(:exec_rollback_to_savepoint)
|
39
|
+
end
|
40
|
+
|
41
|
+
def release_savepoint(*)
|
42
|
+
_throw_transaction_exception!(:release_savepoint)
|
43
|
+
end
|
44
|
+
|
45
|
+
private
|
46
|
+
|
47
|
+
def _throw_transaction_exception!(method_name)
|
48
|
+
return unless ElasticsearchRecord.error_on_transaction
|
49
|
+
raise NotImplementedError, "'##{method_name}' is not supported by Elasticsearch.\nTry to prevent transactions or set the 'ElasticsearchRecord.error_on_transaction' to false!"
|
50
|
+
end
|
51
|
+
end
|
52
|
+
end
|
53
|
+
end
|
54
|
+
end
|
@@ -3,13 +3,6 @@
|
|
3
3
|
module ActiveRecord
|
4
4
|
module ConnectionAdapters
|
5
5
|
module Elasticsearch
|
6
|
-
|
7
|
-
class UnsupportedImplementationError < StandardError
|
8
|
-
def initialize(method_name)
|
9
|
-
super "Unsupported implementation of method: #{method_name}."
|
10
|
-
end
|
11
|
-
end
|
12
|
-
|
13
6
|
module UnsupportedImplementation
|
14
7
|
extend ActiveSupport::Concern
|
15
8
|
|
@@ -13,6 +13,7 @@ require 'active_record/connection_adapters/elasticsearch/schema_dumper'
|
|
13
13
|
require 'active_record/connection_adapters/elasticsearch/schema_statements'
|
14
14
|
require 'active_record/connection_adapters/elasticsearch/type'
|
15
15
|
require 'active_record/connection_adapters/elasticsearch/table_statements'
|
16
|
+
require 'active_record/connection_adapters/elasticsearch/transactions'
|
16
17
|
|
17
18
|
require 'arel/visitors/elasticsearch'
|
18
19
|
require 'arel/collectors/elasticsearch_query'
|
@@ -25,6 +26,12 @@ module ActiveRecord # :nodoc:
|
|
25
26
|
def elasticsearch_connection(config)
|
26
27
|
config = config.symbolize_keys
|
27
28
|
|
29
|
+
# move 'username' to 'user'
|
30
|
+
config[:user] = config.delete(:username) if config[:username]
|
31
|
+
|
32
|
+
# append 'port' to 'host'
|
33
|
+
config[:host] += ":#{config.delete(:port)}" if config[:port] && config[:host]
|
34
|
+
|
28
35
|
# move 'host' to 'hosts'
|
29
36
|
config[:hosts] = config.delete(:host) if config[:host]
|
30
37
|
|
@@ -45,7 +52,7 @@ module ActiveRecord # :nodoc:
|
|
45
52
|
|
46
53
|
# defines the Elasticsearch 'base' structure, which is always included but cannot be resolved through mappings ...
|
47
54
|
BASE_STRUCTURE = [
|
48
|
-
{ 'name' => '_id', 'type' => 'keyword', '
|
55
|
+
{ 'name' => '_id', 'type' => 'keyword', 'meta' => { 'primary_key' => 'true' } },
|
49
56
|
{ 'name' => '_index', 'type' => 'keyword', 'virtual' => true },
|
50
57
|
{ 'name' => '_score', 'type' => 'float', 'virtual' => true },
|
51
58
|
{ 'name' => '_type', 'type' => 'keyword', 'virtual' => true },
|
@@ -57,6 +64,7 @@ module ActiveRecord # :nodoc:
|
|
57
64
|
include Elasticsearch::DatabaseStatements
|
58
65
|
include Elasticsearch::SchemaStatements
|
59
66
|
include Elasticsearch::TableStatements
|
67
|
+
include Elasticsearch::Transactions
|
60
68
|
|
61
69
|
class << self
|
62
70
|
def base_structure_keys
|
@@ -69,7 +77,7 @@ module ActiveRecord # :nodoc:
|
|
69
77
|
client.ping unless config[:ping] == false
|
70
78
|
client
|
71
79
|
rescue ::Elastic::Transport::Transport::Errors::Unauthorized
|
72
|
-
raise ActiveRecord::DatabaseConnectionError.username_error(config[:
|
80
|
+
raise ActiveRecord::DatabaseConnectionError.username_error(config[:user])
|
73
81
|
rescue ::Elastic::Transport::Transport::ServerError => error
|
74
82
|
raise ::ActiveRecord::ConnectionNotEstablished, error.message
|
75
83
|
end
|
@@ -135,7 +143,7 @@ module ActiveRecord # :nodoc:
|
|
135
143
|
|
136
144
|
# define native types - which will be used for schema-dumping
|
137
145
|
NATIVE_DATABASE_TYPES = {
|
138
|
-
primary_key: { name: 'long' },
|
146
|
+
primary_key: { name: 'long' }, # maybe this hae to changed to 'keyword'
|
139
147
|
string: { name: 'keyword' },
|
140
148
|
blob: { name: 'binary' },
|
141
149
|
datetime: { name: 'date' },
|
@@ -172,6 +180,12 @@ module ActiveRecord # :nodoc:
|
|
172
180
|
@config[:migrations_paths] || ['db/migrate_elasticsearch']
|
173
181
|
end
|
174
182
|
|
183
|
+
# Does this adapter support transactions in general?
|
184
|
+
# HINT: This is +NOT* an official setting and only introduced to ElasticsearchRecord
|
185
|
+
def supports_transactions?
|
186
|
+
false
|
187
|
+
end
|
188
|
+
|
175
189
|
# Does this adapter support explain?
|
176
190
|
def supports_explain?
|
177
191
|
false
|
@@ -8,7 +8,7 @@ module ElasticsearchRecord
|
|
8
8
|
# this through +_read_attribute(:id)+.
|
9
9
|
# To also have the ability of accessing this attribute through the default, this flag can be enabled.
|
10
10
|
# @attribute! Boolean
|
11
|
-
class_attribute :delegate_id_attribute,
|
11
|
+
class_attribute :delegate_id_attribute, default: false
|
12
12
|
|
13
13
|
# Elasticsearch's default value for queries without a +size+ is forced to +10+.
|
14
14
|
# To provide a similar behaviour as SQL, this can be automatically set to the +max_result_window+ value.
|
@@ -45,7 +45,7 @@ module ElasticsearchRecord
|
|
45
45
|
|
46
46
|
# overwrite to provide a Elasticsearch version of returning a 'primary_key' was attribute.
|
47
47
|
# Elasticsearch uses the static +_id+ column as primary_key, but also supports an additional +id+ column.
|
48
|
-
# To provide functionality of returning the +
|
48
|
+
# To provide functionality of returning the +id_was+ attribute, this method must also support it
|
49
49
|
# with enabled +delegate_id_attribute+.
|
50
50
|
def id_was
|
51
51
|
delegate_id_attribute? && has_attribute?('id') ? attribute_was('id') : super
|
@@ -69,6 +69,19 @@ module ElasticsearchRecord
|
|
69
69
|
super
|
70
70
|
end
|
71
71
|
|
72
|
+
# resets a possible active +delegate_id_attribute?+ to false during block execution.
|
73
|
+
# Unfortunately this is required, since a lot of rails-code forces 'accessors' on the primary_key-field through the
|
74
|
+
# +id+-getter & setter methods. This will then fail to set the doc-_id and instead set the +id+-attribute ...
|
75
|
+
def undelegate_id_attribute_with(&block)
|
76
|
+
return block.call unless self.delegate_id_attribute?
|
77
|
+
|
78
|
+
self.delegate_id_attribute = false
|
79
|
+
result = block.call
|
80
|
+
self.delegate_id_attribute = true
|
81
|
+
|
82
|
+
result
|
83
|
+
end
|
84
|
+
|
72
85
|
module PrependClassMethods
|
73
86
|
# returns the table_name.
|
74
87
|
# Has to be prepended to provide automated compatibility to other gems.
|
@@ -46,7 +46,7 @@ module ElasticsearchRecord
|
|
46
46
|
|
47
47
|
# final coloring
|
48
48
|
name = color(name, name_color(payload[:name]), true)
|
49
|
-
query = color(query, gate_color(payload[:gate]), true) if colorize_logging
|
49
|
+
query = color(query, gate_color(payload[:gate], payload[:name]), true) if colorize_logging
|
50
50
|
|
51
51
|
debug " #{name} #{query.presence || '-/-'}"
|
52
52
|
end
|
@@ -61,7 +61,7 @@ module ElasticsearchRecord
|
|
61
61
|
end
|
62
62
|
end
|
63
63
|
|
64
|
-
def gate_color(gate)
|
64
|
+
def gate_color(gate, name)
|
65
65
|
case gate
|
66
66
|
# SELECTS
|
67
67
|
when 'core.get', 'core.mget', 'core.search', 'core.msearch', 'core.count', 'core.exists', 'sql.query'
|
@@ -77,7 +77,11 @@ module ElasticsearchRecord
|
|
77
77
|
YELLOW
|
78
78
|
# MIXINS
|
79
79
|
when /indices\.\w+/, 'core.bulk', 'core.index'
|
80
|
-
|
80
|
+
if name.end_with?('Pit Delete')
|
81
|
+
RED
|
82
|
+
else
|
83
|
+
WHITE
|
84
|
+
end
|
81
85
|
else
|
82
86
|
MAGENTA
|
83
87
|
end
|
@@ -8,9 +8,6 @@ module ElasticsearchRecord
|
|
8
8
|
@klass = klass
|
9
9
|
end
|
10
10
|
|
11
|
-
# undelegated schema methods: clone rename create
|
12
|
-
# those should not be quick-accessible, since they might end in heavily broken index
|
13
|
-
|
14
11
|
# delegated dangerous methods (created with exclamation mark)
|
15
12
|
# not able to provide individual arguments - always the defaults will be used!
|
16
13
|
#
|
@@ -26,6 +23,21 @@ module ElasticsearchRecord
|
|
26
23
|
end
|
27
24
|
end
|
28
25
|
|
26
|
+
# delegated dangerous methods with args
|
27
|
+
#
|
28
|
+
# @example
|
29
|
+
# create!(:new_table_name, settings: , mappings:, alias: , ...)
|
30
|
+
# clone!(:new_table_name)
|
31
|
+
# rename!(:new_table_name)
|
32
|
+
# backup!(to: :backup_name)
|
33
|
+
# restore!(from: :backup_name)
|
34
|
+
# reindex!(:new_table_name)
|
35
|
+
%w(create clone rename backup restore reindex).each do |method|
|
36
|
+
define_method("#{method}!") do |*args|
|
37
|
+
_connection.send("#{method}_table", _index_name, *args)
|
38
|
+
end
|
39
|
+
end
|
40
|
+
|
29
41
|
# delegated dangerous methods with confirm parameter (created with exclamation mark)
|
30
42
|
# a exception will be raised, if +confirm:true+ is missing.
|
31
43
|
#
|
@@ -146,12 +158,51 @@ module ElasticsearchRecord
|
|
146
158
|
# Shortcut for meta_exists
|
147
159
|
# @return [Boolean]
|
148
160
|
|
161
|
+
# @!method create!(force: false, copy_from: nil, if_not_exists: false, **options)
|
162
|
+
# Shortcut for create_table
|
163
|
+
# @param [Boolean] force
|
164
|
+
# @param [nil, String] copy_from
|
165
|
+
# @param [Hash] options
|
166
|
+
# @return [Boolean] acknowledged status
|
167
|
+
|
168
|
+
# @!method clone!(target_name, **options)
|
169
|
+
# Shortcut for clone_table
|
170
|
+
# @param [String] target_name
|
171
|
+
# @param [Hash] options
|
172
|
+
# @return [Boolean]
|
173
|
+
|
174
|
+
# @!method rename!(target_name, timeout: nil, **options)
|
175
|
+
# Shortcut for rename_table
|
176
|
+
# @param [String] target_name
|
177
|
+
# @param [String (frozen)] timeout
|
178
|
+
# @param [Hash] options
|
179
|
+
|
180
|
+
# @!method backup!(to: nil, close: true)
|
181
|
+
# Shortcut for backup_table
|
182
|
+
# @param [String] to
|
183
|
+
# @param [Boolean] close
|
184
|
+
# @return [String] backup_name
|
185
|
+
|
186
|
+
# @!method restore!(from:, timeout: nil, open: true, drop_backup: false)
|
187
|
+
# Shortcut for restore_table
|
188
|
+
# @param [String] from
|
189
|
+
# @param [String (frozen)] timeout
|
190
|
+
# @param [Boolean] open
|
191
|
+
# @return [Boolean] acknowledged status
|
192
|
+
|
193
|
+
# @!method reindex!(target_name, **options)
|
194
|
+
# Shortcut for reindex_table
|
195
|
+
# @param [String] target_name
|
196
|
+
# @param [Hash] options
|
197
|
+
# @return [Hash] reindex stats
|
198
|
+
|
149
199
|
# fast insert/update data.
|
200
|
+
# IMPORTANT: Any 'doc'-id must by provided with underscore '_' ( +:_id+ )
|
150
201
|
#
|
151
202
|
# @example
|
152
203
|
# index([{name: 'Hans', age: 34}, {name: 'Peter', age: 22}])
|
153
204
|
#
|
154
|
-
# index({
|
205
|
+
# index({_id: 5, name: 'Georg', age: 87})
|
155
206
|
#
|
156
207
|
# @param [Array<Hash>,Hash] data
|
157
208
|
# @param [Hash] options
|
@@ -160,6 +211,7 @@ module ElasticsearchRecord
|
|
160
211
|
end
|
161
212
|
|
162
213
|
# fast insert new data.
|
214
|
+
# IMPORTANT: Any 'doc'-id must by provided with underscore '_' ( +:_id+ )
|
163
215
|
#
|
164
216
|
# @example
|
165
217
|
# insert([{name: 'Hans', age: 34}, {name: 'Peter', age: 22}])
|
@@ -173,11 +225,12 @@ module ElasticsearchRecord
|
|
173
225
|
end
|
174
226
|
|
175
227
|
# fast update existing data.
|
228
|
+
# IMPORTANT: Any 'doc'-id must by provided with underscore '_' ( +:_id+ )
|
176
229
|
#
|
177
230
|
# @example
|
178
|
-
# update([{
|
231
|
+
# update([{_id: 1, name: 'Hansi'}, {_id: 2, name: 'Peter Parker', age: 42}])
|
179
232
|
#
|
180
|
-
# update({
|
233
|
+
# update({_id: 3, name: 'Georg McCain'})
|
181
234
|
#
|
182
235
|
# @param [Array<Hash>,Hash] data
|
183
236
|
# @param [Hash] options
|
@@ -186,13 +239,14 @@ module ElasticsearchRecord
|
|
186
239
|
end
|
187
240
|
|
188
241
|
# fast delete data.
|
242
|
+
# IMPORTANT: Any 'doc'-id must by provided with underscore '_' ( +:_id+ )
|
189
243
|
#
|
190
244
|
# @example
|
191
245
|
# delete([1,2,3,5])
|
192
246
|
#
|
193
247
|
# delete(3)
|
194
248
|
#
|
195
|
-
# delete({
|
249
|
+
# delete({_id: 2})
|
196
250
|
#
|
197
251
|
# @param [Array<Hash>,Hash] data
|
198
252
|
# @param [Hash] options
|
@@ -202,12 +256,12 @@ module ElasticsearchRecord
|
|
202
256
|
if data[0].is_a?(Hash)
|
203
257
|
bulk(data, :delete, **options)
|
204
258
|
else
|
205
|
-
bulk(data.map { |id| {
|
259
|
+
bulk(data.map { |id| { _id: id } }, :delete, **options)
|
206
260
|
end
|
207
261
|
end
|
208
262
|
|
209
263
|
# bulk handle provided data (single Hash or multiple Array<Hash>).
|
210
|
-
# @param [Hash,Array<Hash
|
264
|
+
# @param [Hash,Array<Hash<Symbol=>Object>>] data - the data to insert/update/delete ...
|
211
265
|
# @param [Symbol] operation
|
212
266
|
# @param [Boolean, Symbol] refresh
|
213
267
|
def bulk(data, operation = :index, refresh: true, **options)
|
@@ -215,7 +269,11 @@ module ElasticsearchRecord
|
|
215
269
|
|
216
270
|
_connection.api(:core, :bulk, {
|
217
271
|
index: _index_name,
|
218
|
-
body:
|
272
|
+
body: if operation == :update
|
273
|
+
data.map { |item| { operation => { _id: (item[:_id].presence || item['_id']), data: { doc: item.except(:_id, '_id') } } } }
|
274
|
+
else
|
275
|
+
data.map { |item| { operation => { _id: (item[:_id].presence || item['_id']), data: item.except(:_id, '_id') } } }
|
276
|
+
end,
|
219
277
|
refresh: refresh
|
220
278
|
}, "BULK #{operation.to_s.upcase}", **options)
|
221
279
|
end
|
@@ -52,11 +52,8 @@ module ElasticsearchRecord
|
|
52
52
|
# @return [Array<String>]
|
53
53
|
def searchable_column_names
|
54
54
|
@searchable_column_names ||= columns.select(&:enabled?).reduce([]) { |m, column|
|
55
|
-
m
|
56
|
-
|
57
|
-
m += column.property_names
|
58
|
-
m.uniq
|
59
|
-
}
|
55
|
+
m + [column.name] + column.field_names + column.property_names
|
56
|
+
}.uniq
|
60
57
|
end
|
61
58
|
|
62
59
|
# clears schema-related instance variables.
|
@@ -11,7 +11,7 @@ module ElasticsearchRecord
|
|
11
11
|
# values is not a "key=>values"-Hash, but a +ActiveModel::Attribute+ - so the casted values gets resolved here
|
12
12
|
values = values.transform_values(&:value)
|
13
13
|
|
14
|
-
# resolve & update a auto_increment value
|
14
|
+
# resolve & update a auto_increment value, if configured
|
15
15
|
_insert_with_auto_increment(values) do |arguments|
|
16
16
|
# build new query
|
17
17
|
query = ElasticsearchRecord::Query.new(
|
@@ -68,6 +68,9 @@ module ElasticsearchRecord
|
|
68
68
|
if (id = values[self.primary_key]).present?
|
69
69
|
yield({id: id})
|
70
70
|
elsif auto_increment?
|
71
|
+
# future increments: uuid (+uuidv6 ?), hex, radix(2-36), integer
|
72
|
+
# allocated through: primary_key_type
|
73
|
+
|
71
74
|
ids = [
|
72
75
|
# try to resolve the current-auto-increment value from the tables meta
|
73
76
|
connection.table_metas(self.table_name).dig('auto_increment').to_i + 1,
|
@@ -88,5 +91,14 @@ module ElasticsearchRecord
|
|
88
91
|
end
|
89
92
|
end
|
90
93
|
end
|
94
|
+
|
95
|
+
# overwrite to provide a Elasticsearch version:
|
96
|
+
# Creates a record with values matching those of the instance attributes
|
97
|
+
# and returns its id.
|
98
|
+
def _create_record(*args)
|
99
|
+
undelegate_id_attribute_with do
|
100
|
+
super
|
101
|
+
end
|
102
|
+
end
|
91
103
|
end
|
92
104
|
end
|
@@ -86,9 +86,9 @@ module ElasticsearchRecord
|
|
86
86
|
# @!attribute Boolean
|
87
87
|
attr_reader :refresh
|
88
88
|
|
89
|
-
# defines the query
|
90
|
-
# @!attribute
|
91
|
-
|
89
|
+
# defines the query timeout
|
90
|
+
# @!attribute Integer|String
|
91
|
+
attr_reader :timeout
|
92
92
|
|
93
93
|
# defines the query arguments to be passed to the API
|
94
94
|
# @!attribute Hash
|
@@ -98,11 +98,12 @@ module ElasticsearchRecord
|
|
98
98
|
# @!attribute Array
|
99
99
|
attr_reader :columns
|
100
100
|
|
101
|
-
def initialize(index: nil, type: TYPE_UNDEFINED, status: STATUS_VALID, body: nil, refresh: nil, arguments: {}, columns: [])
|
101
|
+
def initialize(index: nil, type: TYPE_UNDEFINED, status: STATUS_VALID, body: nil, refresh: nil, timeout: nil, arguments: {}, columns: [])
|
102
102
|
@index = index
|
103
103
|
@type = type
|
104
104
|
@status = status
|
105
105
|
@refresh = refresh
|
106
|
+
@timeout = timeout
|
106
107
|
@body = body
|
107
108
|
@arguments = arguments
|
108
109
|
@columns = columns
|
@@ -163,6 +164,9 @@ module ElasticsearchRecord
|
|
163
164
|
# set refresh, if defined (also includes false value)
|
164
165
|
args[:refresh] = self.refresh unless self.refresh.nil?
|
165
166
|
|
167
|
+
# set timeout, if present
|
168
|
+
args[:timeout] = self.timeout if self.timeout.present?
|
169
|
+
|
166
170
|
args
|
167
171
|
end
|
168
172
|
|
@@ -125,6 +125,18 @@ module ElasticsearchRecord
|
|
125
125
|
self
|
126
126
|
end
|
127
127
|
end
|
128
|
+
|
129
|
+
# overwrite original methods to provide a elasticsearch version:
|
130
|
+
# checks against the +#access_id_fielddata?+ to ensure the Elasticsearch Cluster allows access on the +_id+ field.
|
131
|
+
def reverse_sql_order(order_query)
|
132
|
+
if order_query.empty?
|
133
|
+
return [table[primary_key].desc] if primary_key != '_id' || klass.connection.access_id_fielddata?
|
134
|
+
raise ActiveRecord::IrreversibleOrderError,
|
135
|
+
"Relation has no current order and fielddata access on the _id field is disallowed! However, you can re-enable it by updating the dynamic cluster setting: indices.id_field_data.enabled"
|
136
|
+
end
|
137
|
+
|
138
|
+
super
|
139
|
+
end
|
128
140
|
end
|
129
141
|
end
|
130
142
|
end
|
@@ -102,6 +102,16 @@ module ElasticsearchRecord
|
|
102
102
|
configure!(:__query__, refresh: value)
|
103
103
|
end
|
104
104
|
|
105
|
+
# sets the query's +timeout+ value.
|
106
|
+
# @param [Boolean] value (default: true)
|
107
|
+
def timeout(value = true)
|
108
|
+
spawn.timeout!(value)
|
109
|
+
end
|
110
|
+
|
111
|
+
def timeout!(value = true)
|
112
|
+
configure!(:__query__, timeout: value)
|
113
|
+
end
|
114
|
+
|
105
115
|
# add a whole query 'node' to the query.
|
106
116
|
# @example
|
107
117
|
# query(:bool, {filter: ...})
|
@@ -90,7 +90,9 @@ module ElasticsearchRecord
|
|
90
90
|
#
|
91
91
|
# @param [String] keep_alive - how long to keep alive (for each single request) - default: '1m'
|
92
92
|
# @param [Integer] batch_size - how many results per query (default: 1000 - this means at least 10 queries before reaching the +max_result_window+)
|
93
|
-
|
93
|
+
# @param [Boolean] ids_only - resolve ids only from results
|
94
|
+
# @return [Integer, Array] either returns the results-array (no block provided) or the total amount of results
|
95
|
+
def pit_results(keep_alive: '1m', batch_size: 1000, ids_only: false)
|
94
96
|
raise(ArgumentError, "Batch size cannot be above the 'max_result_window' (#{klass.max_result_window}) !") if batch_size > klass.max_result_window
|
95
97
|
|
96
98
|
# check if limit or offset values where provided
|
@@ -105,6 +107,9 @@ module ElasticsearchRecord
|
|
105
107
|
# see @ https://www.elastic.co/guide/en/elasticsearch/reference/current/paginate-search-results.html
|
106
108
|
relation.order!(_shard_doc: :asc) if relation.order_values.empty? && klass.connection.access_shard_doc?
|
107
109
|
|
110
|
+
# resolve ids only
|
111
|
+
relation.reselect!('_id') if ids_only
|
112
|
+
|
108
113
|
# clear limit & offset
|
109
114
|
relation.offset!(nil).limit!(nil)
|
110
115
|
|
@@ -122,10 +127,16 @@ module ElasticsearchRecord
|
|
122
127
|
# resolve new data until we got all we need
|
123
128
|
loop do
|
124
129
|
# change pit settings & limit (spawn is required, since a +resolve+ will make the relation immutable)
|
125
|
-
current_response = relation.spawn.configure!(current_pit_hash).limit!(batch_size).resolve('Pit').response
|
130
|
+
current_response = relation.spawn.configure!(current_pit_hash).limit!(batch_size).resolve('Pit Results').response
|
126
131
|
|
127
132
|
# resolve only data from hits->hits[{_source}]
|
128
|
-
current_results =
|
133
|
+
current_results = if ids_only
|
134
|
+
current_response['hits']['hits'].map { |result| result['_id'] }
|
135
|
+
# future with helper
|
136
|
+
# current_response['hits']['hits'].map.from_hash('_id')
|
137
|
+
else
|
138
|
+
current_response['hits']['hits'].map { |result| result['_source'].merge('_id' => result['_id']) }
|
139
|
+
end
|
129
140
|
current_results_length = current_results.length
|
130
141
|
|
131
142
|
# check if we reached the required offset
|
@@ -171,12 +182,38 @@ module ElasticsearchRecord
|
|
171
182
|
end
|
172
183
|
end
|
173
184
|
|
174
|
-
# return results array
|
175
|
-
|
185
|
+
# return results array or total value
|
186
|
+
if block_given?
|
187
|
+
results_total
|
188
|
+
else
|
189
|
+
results
|
190
|
+
end
|
176
191
|
end
|
177
192
|
|
178
193
|
alias_method :total_results, :pit_results
|
179
194
|
|
195
|
+
# executes a delete query in a +point_in_time+ scope.
|
196
|
+
# this will provide the possibility to delete more than the +max_result_window+ (default: 10000) docs in a batched process.
|
197
|
+
# @param [String] keep_alive
|
198
|
+
# @param [Integer] batch_size
|
199
|
+
# @param [Boolean] refresh index after delete finished (default: true)
|
200
|
+
# @return [Integer] total amount of deleted docs
|
201
|
+
def pit_delete(keep_alive: '1m', batch_size: 1000, refresh: true)
|
202
|
+
delete_count = select('_id').pit_results(keep_alive: keep_alive, batch_size: batch_size, ids_only: true) do |ids|
|
203
|
+
# skip empty results
|
204
|
+
next unless ids.any?
|
205
|
+
|
206
|
+
# delete all IDs, but do not refresh index, yet
|
207
|
+
klass.connection.api(:core, :bulk, { index: klass.table_name, body: ids.map { |id| { delete: { _id: id } } }, refresh: false }, "#{klass} Pit Delete")
|
208
|
+
end
|
209
|
+
|
210
|
+
# refresh index
|
211
|
+
klass.connection.refresh_table(klass.table_name) if refresh
|
212
|
+
|
213
|
+
# return total count
|
214
|
+
delete_count
|
215
|
+
end
|
216
|
+
|
180
217
|
# returns the RAW response for the current query
|
181
218
|
# @return [Array]
|
182
219
|
def response
|
@@ -49,7 +49,7 @@ module ElasticsearchRecord
|
|
49
49
|
end
|
50
50
|
|
51
51
|
# Returns the RAW +_source+ data from each hit - aka. +rows+.
|
52
|
-
# PLEASE NOTE: The array will only contain the RAW data from each +_source+ (meta info like '_score'
|
52
|
+
# PLEASE NOTE: The array will only contain the RAW data from each +_source+ (meta info like '_id' or '_score' are not included)
|
53
53
|
# @return [Array]
|
54
54
|
def results
|
55
55
|
return [] unless response['hits']
|
data/lib/elasticsearch_record.rb
CHANGED
@@ -55,6 +55,16 @@ module ElasticsearchRecord
|
|
55
55
|
|
56
56
|
autoload :ElasticsearchDatabaseTasks, 'elasticsearch_record/tasks/elasticsearch_database_tasks'
|
57
57
|
end
|
58
|
+
|
59
|
+
##
|
60
|
+
# :singleton-method:
|
61
|
+
# Specifies if a exception should be raised while using transactions.
|
62
|
+
# Since ActiveRecord does not have any configuration option to support transactions and
|
63
|
+
# Elasticsearch does **NOT** support transactions, it may be risky to ignore them.
|
64
|
+
# As default, transactional are 'silently swallowed' to not break any existing applications...
|
65
|
+
# However enabling this flag will surely fail transactional tests ...
|
66
|
+
singleton_class.attr_accessor :error_on_transaction
|
67
|
+
self.error_on_transaction = false
|
58
68
|
end
|
59
69
|
|
60
70
|
ActiveSupport.on_load(:active_record) do
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: elasticsearch_record
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.
|
4
|
+
version: 1.6.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Tobias Gonsior
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2023-
|
11
|
+
date: 2023-08-11 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: activerecord
|
@@ -145,6 +145,7 @@ files:
|
|
145
145
|
- lib/active_record/connection_adapters/elasticsearch/schema_dumper.rb
|
146
146
|
- lib/active_record/connection_adapters/elasticsearch/schema_statements.rb
|
147
147
|
- lib/active_record/connection_adapters/elasticsearch/table_statements.rb
|
148
|
+
- lib/active_record/connection_adapters/elasticsearch/transactions.rb
|
148
149
|
- lib/active_record/connection_adapters/elasticsearch/type.rb
|
149
150
|
- lib/active_record/connection_adapters/elasticsearch/type/format_string.rb
|
150
151
|
- lib/active_record/connection_adapters/elasticsearch/type/multicast_value.rb
|