elasticsearch_record 1.3.1 → 1.4.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/README.md +41 -6
- data/docs/CHANGELOG.md +6 -0
- data/lib/active_record/connection_adapters/elasticsearch_adapter.rb +21 -12
- data/lib/elasticsearch_record/core.rb +5 -5
- data/lib/elasticsearch_record/gem_version.rb +2 -2
- data/lib/elasticsearch_record/instrumentation/log_subscriber.rb +5 -4
- data/lib/elasticsearch_record/model_api.rb +120 -0
- data/lib/elasticsearch_record/result.rb +1 -1
- data/lib/elasticsearch_record.rb +1 -0
- metadata +8 -8
- data/Gemfile.lock +0 -73
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 8a386dd571b2e88a62e26f815c16783d3fda97ed911b4edffa4db884230dfc18
|
4
|
+
data.tar.gz: a50697c2043a3160c31716d07d9afbe05bab691267773b92f424bead64193ecb
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 4c3172895b50805a17964e0db2406b90fd5797c40d91d17ee9c0cea2a26d0b52d328663f5b44ff882d32bf490662146ef5064040361ae5a677d97868078332b5
|
7
|
+
data.tar.gz: 3d86fb89924f99c2ad64e44c225f78c434cabcbadc4660dfa8032c5038394b05b5ab949abd93d9895b3a7148c07f12ac99374c8cee1c2ce805f2188eab6be143
|
data/README.md
CHANGED
@@ -55,10 +55,6 @@ Or install it yourself as:
|
|
55
55
|
* logs Elasticsearch API-calls
|
56
56
|
* shows Runtime in logs
|
57
57
|
|
58
|
-
## Contra - what it _(currently)_ can not
|
59
|
-
* Joins to other indexes or databases
|
60
|
-
* complex, combined or nested queries ```and, or, Model.arel ...```
|
61
|
-
|
62
58
|
## Setup
|
63
59
|
|
64
60
|
### a) Update your **database.yml** and add a elasticsearch connection:
|
@@ -255,12 +251,51 @@ _see simple documentation about these methods @ [rubydoc](https://rubydoc.info/g
|
|
255
251
|
- find_by_query
|
256
252
|
- msearch
|
257
253
|
|
254
|
+
### Useful model API methods
|
255
|
+
Fast access to model-related methods for easier access without creating a overcomplicated method call.
|
256
|
+
|
257
|
+
Access these methods through the model class method `.api`.
|
258
|
+
```ruby
|
259
|
+
# returns mapping of model class
|
260
|
+
klass.api.mappings
|
261
|
+
|
262
|
+
# e.g. for ElasticUser model
|
263
|
+
ElasticUser.api.mappings
|
264
|
+
|
265
|
+
# insert new raw data
|
266
|
+
ElasticUser.api.insert([{name: 'Hans', age: 34}, {name: 'Peter', age: 22}])
|
267
|
+
```
|
268
|
+
|
269
|
+
* open
|
270
|
+
* close
|
271
|
+
* refresh
|
272
|
+
* block
|
273
|
+
* unblock
|
274
|
+
* mappings
|
275
|
+
* metas
|
276
|
+
* settings
|
277
|
+
* aliases
|
278
|
+
* state
|
279
|
+
* schema
|
280
|
+
* exists?
|
281
|
+
* alias_exists?
|
282
|
+
* setting_exists?
|
283
|
+
* mapping_exists?
|
284
|
+
* meta_exists?
|
285
|
+
|
286
|
+
Fast insert, update, delete raw data
|
287
|
+
* index
|
288
|
+
* insert
|
289
|
+
* update
|
290
|
+
* delete
|
291
|
+
* bulk
|
292
|
+
|
258
293
|
## ActiveRecord ConnectionAdapters table-methods
|
259
|
-
Access these methods through the model
|
294
|
+
Access these methods through the model class method `.connection`.
|
260
295
|
|
261
296
|
```ruby
|
262
297
|
# returns mapping of provided table (index)
|
263
|
-
|
298
|
+
klass.connection.table_mappings('table-name')
|
264
299
|
```
|
265
300
|
|
266
301
|
- table_mappings
|
data/docs/CHANGELOG.md
CHANGED
@@ -1,5 +1,11 @@
|
|
1
1
|
# ElasticsearchRecord - CHANGELOG
|
2
2
|
|
3
|
+
## [1.4.0] - 2023-01-27
|
4
|
+
* [add] `ElasticsearchRecord::ModelApi` for fast & easy access the elasticsearch index - callable through `.api` (e.g. ElasticUser.api.mappings)
|
5
|
+
* [ref] `ElasticsearchRecord::Instrumentation::LogSubscriber` to truncate the query-string (default: 1000)
|
6
|
+
* [ref] `ActiveRecord::ConnectionAdapters::ElasticsearchAdapter#log` with extra attribute (log: true) to prevent logging (e.g. on custom api calls)
|
7
|
+
* [fix] `ElasticsearchRecord::Result#bucket` to prevent resolving additional meta key (key_as_string)
|
8
|
+
|
3
9
|
## [1.3.1] - 2023-01-18
|
4
10
|
* [fix] `#none!` method to correctly invalidate the query (String(s) in where-queries like '1=0' will raise now)
|
5
11
|
* [fix] missing 'ChangeSettingDefinition' & 'RemoveSettingDefinition' @ `ActiveRecord::ConnectionAdapters::Elasticsearch::UpdateTableDefinition::COMPOSITE_DEFINITIONS` to composite in a single query
|
@@ -216,14 +216,15 @@ module ActiveRecord # :nodoc:
|
|
216
216
|
# @param [Hash] arguments - action arguments
|
217
217
|
# @param [String (frozen)] name - the logging name
|
218
218
|
# @param [Boolean] async - send async (default: false) - currently not supported
|
219
|
+
# @param [Boolean] log - send log to instrumenter (default: true)
|
219
220
|
# @return [Elasticsearch::API::Response, Object]
|
220
|
-
def api(namespace, action, arguments = {}, name = 'API', async: false)
|
221
|
+
def api(namespace, action, arguments = {}, name = 'API', async: false, log: true)
|
221
222
|
raise ::StandardError, 'ASYNC api calls are not supported' if async
|
222
223
|
|
223
224
|
# resolve the API target
|
224
225
|
target = namespace == :core ? @connection : @connection.__send__(namespace)
|
225
226
|
|
226
|
-
log("#{namespace}.#{action}", arguments, name, async: async) do
|
227
|
+
log("#{namespace}.#{action}", arguments, name, async: async, log: log) do
|
227
228
|
response = ActiveSupport::Dependencies.interlock.permit_concurrent_loads do
|
228
229
|
target.__send__(action, arguments)
|
229
230
|
end
|
@@ -274,16 +275,24 @@ module ActiveRecord # :nodoc:
|
|
274
275
|
end
|
275
276
|
|
276
277
|
# provide a custom log instrumenter for elasticsearch subscribers
|
277
|
-
def log(gate, arguments, name, async: false, &block)
|
278
|
-
|
279
|
-
|
280
|
-
|
281
|
-
|
282
|
-
|
283
|
-
|
284
|
-
|
285
|
-
|
286
|
-
|
278
|
+
def log(gate, arguments, name, async: false, log: true, &block)
|
279
|
+
if log
|
280
|
+
@instrumenter.instrument(
|
281
|
+
"query.elasticsearch_record",
|
282
|
+
gate: gate,
|
283
|
+
name: name,
|
284
|
+
arguments: gate == 'core.msearch' ? arguments.deep_dup : arguments,
|
285
|
+
async: async) do
|
286
|
+
@lock.synchronize(&block)
|
287
|
+
rescue => e
|
288
|
+
raise translate_exception_class(e, arguments, [])
|
289
|
+
end
|
290
|
+
else
|
291
|
+
begin
|
292
|
+
@lock.synchronize(&block)
|
293
|
+
rescue => e
|
294
|
+
raise translate_exception_class(e, arguments, [])
|
295
|
+
end
|
287
296
|
end
|
288
297
|
end
|
289
298
|
|
@@ -3,7 +3,6 @@ module ElasticsearchRecord
|
|
3
3
|
extend ActiveSupport::Concern
|
4
4
|
|
5
5
|
included do
|
6
|
-
|
7
6
|
# Rails resolves the primary_key's value by accessing the +#id+ method.
|
8
7
|
# Since Elasticsearch also supports an additional, independent +id+ attribute, it would only be able to access
|
9
8
|
# this through +read_attribute(:id)+.
|
@@ -84,10 +83,11 @@ module ElasticsearchRecord
|
|
84
83
|
cache.compute_if_absent(key) { ElasticsearchRecord::StatementCache.create(connection, &block) }
|
85
84
|
end
|
86
85
|
|
87
|
-
#
|
88
|
-
#
|
89
|
-
|
90
|
-
|
86
|
+
# used to provide fast access to the connection API without explicit providing table-related parameters.
|
87
|
+
# @return [anonymous Struct]
|
88
|
+
def api
|
89
|
+
ElasticsearchRecord::ModelApi.new(self)
|
90
|
+
end
|
91
91
|
|
92
92
|
private
|
93
93
|
|
@@ -5,7 +5,7 @@ module ElasticsearchRecord
|
|
5
5
|
# attach to ElasticsearchRecord related events
|
6
6
|
class LogSubscriber < ActiveSupport::LogSubscriber
|
7
7
|
|
8
|
-
IGNORE_PAYLOAD_NAMES = %w[SCHEMA EXPLAIN]
|
8
|
+
IGNORE_PAYLOAD_NAMES = %w[SCHEMA EXPLAIN EXCLUDE]
|
9
9
|
|
10
10
|
def self.runtime=(value)
|
11
11
|
Thread.current["elasticsearch_record_runtime"] = value
|
@@ -37,17 +37,18 @@ module ElasticsearchRecord
|
|
37
37
|
end
|
38
38
|
name = "CACHE #{name}" if payload[:cached]
|
39
39
|
|
40
|
-
# nice feature: displays the REAL query-time (_qt)
|
40
|
+
# nice feature: displays the REAL query-time from elasticsearch response (_qt)
|
41
|
+
# this is handled through the +::ActiveRecord::ConnectionAdapters::ElasticsearchAdapter#api+ method
|
41
42
|
name = "#{name} (took: #{payload[:arguments][:_qt].round(1)}ms)" if payload[:arguments][:_qt]
|
42
43
|
|
43
44
|
# build query
|
44
|
-
query = payload[:arguments].except(:_qt).inspect.gsub(/:(\w+)=>/, '\1: ').
|
45
|
+
query = payload[:arguments].except(:_qt).inspect.gsub(/:(\w+)=>/, '\1: ').truncate((payload[:truncate] || 1000), omission: color(' (pruned)', RED))
|
45
46
|
|
46
47
|
# final coloring
|
47
48
|
name = color(name, name_color(payload[:name]), true)
|
48
49
|
query = color(query, gate_color(payload[:gate]), true) if colorize_logging
|
49
50
|
|
50
|
-
debug " #{name} #{query}"
|
51
|
+
debug " #{name} #{query.presence || '-/-'}"
|
51
52
|
end
|
52
53
|
|
53
54
|
private
|
@@ -0,0 +1,120 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module ElasticsearchRecord
|
4
|
+
class ModelApi
|
5
|
+
attr_reader :klass
|
6
|
+
|
7
|
+
def initialize(klass)
|
8
|
+
@klass = klass
|
9
|
+
end
|
10
|
+
|
11
|
+
# undelegated schema methods: clone rename create drop
|
12
|
+
# those should not be quick-accessible, since they might end in heavily broken index
|
13
|
+
|
14
|
+
# delegated dangerous methods (created with exclamation mark)
|
15
|
+
# not able to provide individual arguments - always the defaults will be used
|
16
|
+
%w(open close refresh block unblock).each do |method|
|
17
|
+
define_method("#{method}!") do
|
18
|
+
_connection.send("#{method}_table", _index_name)
|
19
|
+
end
|
20
|
+
end
|
21
|
+
|
22
|
+
# delegated table methods
|
23
|
+
%w(mappings metas settings aliases state schema exists?).each do |method|
|
24
|
+
define_method(method) do |*args|
|
25
|
+
_connection.send("table_#{method}", _index_name, *args)
|
26
|
+
end
|
27
|
+
end
|
28
|
+
|
29
|
+
# delegated plain methods
|
30
|
+
%w(alias_exists? setting_exists? mapping_exists? meta_exists?).each do |method|
|
31
|
+
define_method(method) do |*args|
|
32
|
+
_connection.send(method, _index_name, *args)
|
33
|
+
end
|
34
|
+
end
|
35
|
+
|
36
|
+
# fast insert/update data.
|
37
|
+
#
|
38
|
+
# @example
|
39
|
+
# index([{name: 'Hans', age: 34}, {name: 'Peter', age: 22}])
|
40
|
+
#
|
41
|
+
# index({id: 5, name: 'Georg', age: 87})
|
42
|
+
#
|
43
|
+
# @param [Array<Hash>,Hash] data
|
44
|
+
# @param [Hash] options
|
45
|
+
def index(data, **options)
|
46
|
+
bulk(data, :index, **options)
|
47
|
+
end
|
48
|
+
|
49
|
+
# fast insert new data.
|
50
|
+
#
|
51
|
+
# @example
|
52
|
+
# insert([{name: 'Hans', age: 34}, {name: 'Peter', age: 22}])
|
53
|
+
#
|
54
|
+
# insert({name: 'Georg', age: 87})
|
55
|
+
#
|
56
|
+
# @param [Array<Hash>,Hash] data
|
57
|
+
# @param [Hash] options
|
58
|
+
def insert(data, **options)
|
59
|
+
bulk(data, :create, **options)
|
60
|
+
end
|
61
|
+
|
62
|
+
# fast update existing data.
|
63
|
+
#
|
64
|
+
# @example
|
65
|
+
# update([{id: 1, name: 'Hansi'}, {id: 2, name: 'Peter Parker', age: 42}])
|
66
|
+
#
|
67
|
+
# update({id: 3, name: 'Georg McCain'})
|
68
|
+
#
|
69
|
+
# @param [Array<Hash>,Hash] data
|
70
|
+
# @param [Hash] options
|
71
|
+
def update(data, **options)
|
72
|
+
bulk(data, :update, **options)
|
73
|
+
end
|
74
|
+
|
75
|
+
# fast delete data.
|
76
|
+
#
|
77
|
+
# @example
|
78
|
+
# delete([1,2,3,5])
|
79
|
+
#
|
80
|
+
# delete(3)
|
81
|
+
#
|
82
|
+
# delete({id: 2})
|
83
|
+
#
|
84
|
+
# @param [Array<Hash>,Hash] data
|
85
|
+
# @param [Hash] options
|
86
|
+
def delete(data, **options)
|
87
|
+
data = [data] unless data.is_a?(Array)
|
88
|
+
|
89
|
+
if data[0].is_a?(Hash)
|
90
|
+
bulk(data, :delete, **options)
|
91
|
+
else
|
92
|
+
bulk(data.map{|id| {id: id}}, :delete, **options)
|
93
|
+
end
|
94
|
+
end
|
95
|
+
|
96
|
+
# bulk handle provided data (single Hash or multiple Array<Hash>).
|
97
|
+
# @param [Hash,Array<Hash>] data - the data to insert/update/delete ...
|
98
|
+
# @param [Symbol] operation
|
99
|
+
# @param [Boolean, Symbol] refresh
|
100
|
+
def bulk(data, operation = :index, refresh: true, **options)
|
101
|
+
data = [data] unless data.is_a?(Array)
|
102
|
+
|
103
|
+
_connection.api(:core, :bulk, {
|
104
|
+
index: _index_name,
|
105
|
+
body: data.map{|item| {operation => {_id: item[:id], data: item.except(:id)}}},
|
106
|
+
refresh: refresh
|
107
|
+
}, "BULK #{operation.to_s.upcase}", **options)
|
108
|
+
end
|
109
|
+
|
110
|
+
private
|
111
|
+
|
112
|
+
def _index_name
|
113
|
+
klass.index_name
|
114
|
+
end
|
115
|
+
|
116
|
+
def _connection
|
117
|
+
klass.connection
|
118
|
+
end
|
119
|
+
end
|
120
|
+
end
|
@@ -256,7 +256,7 @@ module ElasticsearchRecord
|
|
256
256
|
else
|
257
257
|
# resolve sub-aggregations / nodes without 'meta' keys.
|
258
258
|
# if this results in an empty hash, the return will be nil
|
259
|
-
node.except(:key, :doc_count, :doc_count_error_upper_bound, :sum_other_doc_count).transform_values { |val| _resolve_bucket(val) }.presence
|
259
|
+
node.except(:key, :doc_count, :doc_count_error_upper_bound, :sum_other_doc_count, :key_as_string).transform_values { |val| _resolve_bucket(val) }.presence
|
260
260
|
end
|
261
261
|
end
|
262
262
|
|
data/lib/elasticsearch_record.rb
CHANGED
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: elasticsearch_record
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.
|
4
|
+
version: 1.4.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Tobias Gonsior
|
8
|
-
autorequire:
|
8
|
+
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2023-01-
|
11
|
+
date: 2023-01-27 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: activerecord
|
@@ -97,7 +97,7 @@ dependencies:
|
|
97
97
|
description: 'ElasticsearchRecord is a ActiveRecord adapter and provides similar functionality
|
98
98
|
for Elasticsearch.
|
99
99
|
|
100
|
-
|
100
|
+
'
|
101
101
|
email:
|
102
102
|
- info@ruby-smart.org
|
103
103
|
executables: []
|
@@ -107,7 +107,6 @@ files:
|
|
107
107
|
- ".rspec"
|
108
108
|
- ".yardopts"
|
109
109
|
- Gemfile
|
110
|
-
- Gemfile.lock
|
111
110
|
- README.md
|
112
111
|
- Rakefile
|
113
112
|
- docs/CHANGELOG.md
|
@@ -159,6 +158,7 @@ files:
|
|
159
158
|
- lib/elasticsearch_record/instrumentation/controller_runtime.rb
|
160
159
|
- lib/elasticsearch_record/instrumentation/log_subscriber.rb
|
161
160
|
- lib/elasticsearch_record/instrumentation/railtie.rb
|
161
|
+
- lib/elasticsearch_record/model_api.rb
|
162
162
|
- lib/elasticsearch_record/model_schema.rb
|
163
163
|
- lib/elasticsearch_record/patches/active_record/relation_merger_patch.rb
|
164
164
|
- lib/elasticsearch_record/patches/arel/select_core_patch.rb
|
@@ -191,7 +191,7 @@ metadata:
|
|
191
191
|
source_code_uri: https://github.com/ruby-smart/elasticsearch_record
|
192
192
|
documentation_uri: https://rubydoc.info/gems/elasticsearch_record
|
193
193
|
changelog_uri: https://github.com/ruby-smart/elasticsearch_record/blob/main/docs/CHANGELOG.md
|
194
|
-
post_install_message:
|
194
|
+
post_install_message:
|
195
195
|
rdoc_options: []
|
196
196
|
require_paths:
|
197
197
|
- lib
|
@@ -206,8 +206,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
206
206
|
- !ruby/object:Gem::Version
|
207
207
|
version: '0'
|
208
208
|
requirements: []
|
209
|
-
rubygems_version: 3.
|
210
|
-
signing_key:
|
209
|
+
rubygems_version: 3.1.6
|
210
|
+
signing_key:
|
211
211
|
specification_version: 4
|
212
212
|
summary: ActiveRecord adapter for Elasticsearch
|
213
213
|
test_files: []
|
data/Gemfile.lock
DELETED
@@ -1,73 +0,0 @@
|
|
1
|
-
PATH
|
2
|
-
remote: .
|
3
|
-
specs:
|
4
|
-
elasticsearch_record (1.2.4)
|
5
|
-
activerecord (~> 7.0.0)
|
6
|
-
elasticsearch (~> 8.4)
|
7
|
-
|
8
|
-
GEM
|
9
|
-
remote: https://rubygems.org/
|
10
|
-
specs:
|
11
|
-
activemodel (7.0.4)
|
12
|
-
activesupport (= 7.0.4)
|
13
|
-
activerecord (7.0.4)
|
14
|
-
activemodel (= 7.0.4)
|
15
|
-
activesupport (= 7.0.4)
|
16
|
-
activesupport (7.0.4)
|
17
|
-
concurrent-ruby (~> 1.0, >= 1.0.2)
|
18
|
-
i18n (>= 1.6, < 2)
|
19
|
-
minitest (>= 5.1)
|
20
|
-
tzinfo (~> 2.0)
|
21
|
-
concurrent-ruby (1.1.10)
|
22
|
-
diff-lcs (1.5.0)
|
23
|
-
elastic-transport (8.1.0)
|
24
|
-
faraday (< 3)
|
25
|
-
multi_json
|
26
|
-
elasticsearch (8.5.2)
|
27
|
-
elastic-transport (~> 8)
|
28
|
-
elasticsearch-api (= 8.5.2)
|
29
|
-
elasticsearch-api (8.5.2)
|
30
|
-
multi_json
|
31
|
-
faraday (2.7.2)
|
32
|
-
faraday-net_http (>= 2.0, < 3.1)
|
33
|
-
ruby2_keywords (>= 0.0.4)
|
34
|
-
faraday-net_http (3.0.2)
|
35
|
-
i18n (1.12.0)
|
36
|
-
concurrent-ruby (~> 1.0)
|
37
|
-
minitest (5.16.3)
|
38
|
-
multi_json (1.15.0)
|
39
|
-
rake (13.0.6)
|
40
|
-
rspec (3.11.0)
|
41
|
-
rspec-core (~> 3.11.0)
|
42
|
-
rspec-expectations (~> 3.11.0)
|
43
|
-
rspec-mocks (~> 3.11.0)
|
44
|
-
rspec-core (3.11.0)
|
45
|
-
rspec-support (~> 3.11.0)
|
46
|
-
rspec-expectations (3.11.1)
|
47
|
-
diff-lcs (>= 1.2.0, < 2.0)
|
48
|
-
rspec-support (~> 3.11.0)
|
49
|
-
rspec-mocks (3.11.1)
|
50
|
-
diff-lcs (>= 1.2.0, < 2.0)
|
51
|
-
rspec-support (~> 3.11.0)
|
52
|
-
rspec-support (3.11.1)
|
53
|
-
ruby2_keywords (0.0.5)
|
54
|
-
tzinfo (2.0.5)
|
55
|
-
concurrent-ruby (~> 1.0)
|
56
|
-
webrick (1.7.0)
|
57
|
-
yard (0.9.28)
|
58
|
-
webrick (~> 1.7.0)
|
59
|
-
yard-activesupport-concern (0.0.1)
|
60
|
-
yard (>= 0.8)
|
61
|
-
|
62
|
-
PLATFORMS
|
63
|
-
x86_64-linux
|
64
|
-
|
65
|
-
DEPENDENCIES
|
66
|
-
elasticsearch_record!
|
67
|
-
rake (~> 13.0)
|
68
|
-
rspec (~> 3.0)
|
69
|
-
yard (~> 0.9)
|
70
|
-
yard-activesupport-concern (~> 0.0.1)
|
71
|
-
|
72
|
-
BUNDLED WITH
|
73
|
-
2.3.18
|