elasticsearch 7.13.3 → 8.14.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/Gemfile +8 -6
- data/README.md +71 -33
- data/Rakefile +15 -20
- data/bin/elastic_ruby_console +2 -18
- data/elasticsearch.gemspec +24 -35
- data/lib/elasticsearch/helpers/bulk_helper.rb +129 -0
- data/lib/elasticsearch/helpers/esql_helper.rb +72 -0
- data/lib/elasticsearch/helpers/scroll_helper.rb +95 -0
- data/lib/elasticsearch/version.rb +1 -1
- data/lib/elasticsearch.rb +167 -7
- data/spec/integration/characters_escaping_spec.rb +94 -0
- data/spec/integration/client_integration_spec.rb +64 -0
- data/spec/integration/helpers/bulk_helper_spec.rb +206 -0
- data/spec/integration/helpers/esql_helper_spec.rb +118 -0
- data/spec/integration/helpers/helpers_spec_helper.rb +29 -0
- data/spec/integration/helpers/scroll_helper_spec.rb +81 -0
- data/spec/integration/opentelemetry_spec.rb +55 -0
- data/spec/spec_helper.rb +43 -0
- data/spec/unit/api_key_spec.rb +138 -0
- data/spec/unit/cloud_credentials_spec.rb +167 -0
- data/spec/unit/custom_transport_implementation_spec.rb +43 -0
- data/spec/unit/elasticsearch_product_validation_spec.rb +148 -0
- data/spec/unit/headers_spec.rb +55 -0
- data/spec/unit/opaque_id_spec.rb +48 -0
- data/spec/unit/user_agent_spec.rb +69 -0
- data/{test/unit/wrapper_gem_test.rb → spec/unit/wrapper_gem_spec.rb} +11 -21
- metadata +64 -111
- data/test/integration/client_integration_test.rb +0 -80
- data/test/test_helper.rb +0 -115
- /data/{LICENSE → LICENSE.txt} +0 -0
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 829c510c4fb0664f58d4be87a7d0d8eed2906b68e659f63b33a7e082cba6c1ee
|
4
|
+
data.tar.gz: bafb973a40c4eda2a5de4fb0abe5a7f4308efbe8105f531545953a405d2cb83f
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 4c98e9e0c6cc219b281da17019021ce7bd7e28120c9390b05dd71550bf2527c99570002f779f504f10820effe1fc2cd0b2824213bb445184fb7b77eabc69532b
|
7
|
+
data.tar.gz: 6178b702bd1077554debcf12783ffd1cda149c507c87284cbe60028a0f06fe0db8f2c140f3c29fd19fca23ec456012087ff25f6dcec34553e63d550146248ede
|
data/Gemfile
CHANGED
@@ -1,3 +1,5 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
# Licensed to Elasticsearch B.V. under one or more contributor
|
2
4
|
# license agreements. See the NOTICE file distributed with
|
3
5
|
# this work for additional information regarding copyright
|
@@ -20,14 +22,14 @@ source 'https://rubygems.org'
|
|
20
22
|
# Specify your gem's dependencies in elasticsearch.gemspec
|
21
23
|
gemspec
|
22
24
|
|
23
|
-
if File.exist? File.expand_path(
|
24
|
-
gem 'elasticsearch-api', :
|
25
|
+
if File.exist? File.expand_path('../elasticsearch-api/elasticsearch-api.gemspec', __dir__)
|
26
|
+
gem 'elasticsearch-api', path: File.expand_path('../elasticsearch-api', __dir__), require: false
|
25
27
|
end
|
26
28
|
|
27
|
-
if
|
28
|
-
gem '
|
29
|
+
if ENV['TRANSPORT_VERSION'] == 'main'
|
30
|
+
gem 'elastic-transport', git: 'https://github.com/elastic/elastic-transport-ruby.git', branch: 'main'
|
29
31
|
end
|
30
32
|
|
31
|
-
if
|
32
|
-
gem '
|
33
|
+
if RUBY_VERSION >= '3.0'
|
34
|
+
gem 'opentelemetry-sdk', require: false
|
33
35
|
end
|
data/README.md
CHANGED
@@ -2,6 +2,29 @@
|
|
2
2
|
|
3
3
|
The `elasticsearch` library provides a Ruby client and API for [Elasticsearch](http://elasticsearch.com).
|
4
4
|
|
5
|
+
## Usage
|
6
|
+
|
7
|
+
This gem is a wrapper for two separate libraries:
|
8
|
+
|
9
|
+
* [`elastic-transport`](https://github.com/elastic/elastic-transport-ruby/), which provides a low-level Ruby client for connecting to [Elastic](http://elasticsearch.com) services.
|
10
|
+
* [`elasticsearch-api`](https://github.com/elasticsearch/elasticsearch-ruby/tree/main/elasticsearch-api), which provides a Ruby API for the Elasticsearch RESTful API.
|
11
|
+
|
12
|
+
Install the `elasticsearch` package and use the API directly:
|
13
|
+
|
14
|
+
```ruby
|
15
|
+
require 'elasticsearch'
|
16
|
+
|
17
|
+
client = Elasticsearch::Client.new(log: true)
|
18
|
+
|
19
|
+
client.cluster.health
|
20
|
+
|
21
|
+
client.transport.reload_connections!
|
22
|
+
|
23
|
+
client.search(q: 'test')
|
24
|
+
|
25
|
+
# etc.
|
26
|
+
```
|
27
|
+
|
5
28
|
Features overview:
|
6
29
|
|
7
30
|
* Pluggable logging and tracing
|
@@ -14,25 +37,13 @@ Features overview:
|
|
14
37
|
* Extensive documentation and examples
|
15
38
|
* Emphasis on modularity and extendability of both the client and API libraries
|
16
39
|
|
17
|
-
(For integration with Ruby models and Rails applications,
|
18
|
-
see the <https://github.com/elasticsearch/elasticsearch-rails> project.)
|
40
|
+
(For integration with Ruby models and Rails applications, see the <https://github.com/elasticsearch/elasticsearch-rails> project.)
|
19
41
|
|
20
42
|
## Compatibility
|
21
43
|
|
22
|
-
|
44
|
+
We follow Ruby’s own maintenance policy and officially support all currently maintained versions per [Ruby Maintenance Branches](https://www.ruby-lang.org/en/downloads/branches/).
|
23
45
|
|
24
|
-
|
25
|
-
just use a release matching major version of Elasticsearch.
|
26
|
-
|
27
|
-
| Ruby | | Elasticsearch |
|
28
|
-
|:-------------:|:-:| :-----------: |
|
29
|
-
| 0.90 | → | 0.90 |
|
30
|
-
| 1.x | → | 1.x |
|
31
|
-
| 2.x | → | 2.x |
|
32
|
-
| 5.x | → | 5.x |
|
33
|
-
| 6.x | → | 6.x |
|
34
|
-
| 7.x | → | 7.x |
|
35
|
-
| master | → | master |
|
46
|
+
Language clients are forward compatible; meaning that clients support communicating with greater minor versions of Elasticsearch. Elastic language clients are also backwards compatible with lesser supported minor Elasticsearch versions.
|
36
47
|
|
37
48
|
## Installation
|
38
49
|
|
@@ -51,41 +62,68 @@ or install it from a source code checkout:
|
|
51
62
|
bundle install
|
52
63
|
rake install
|
53
64
|
|
54
|
-
##
|
65
|
+
## Configuration
|
55
66
|
|
56
|
-
|
67
|
+
* [Identifying running tasks with X-Opaque-Id](#identifying-running-tasks-with-x-opaque-id)
|
68
|
+
* [Api Key Authentication](#api-key-authentication)
|
57
69
|
|
58
|
-
|
59
|
-
which provides a low-level Ruby client for connecting to an [Elasticsearch](http://elasticsearch.com) cluster
|
60
|
-
* [`elasticsearch-api`](https://github.com/elasticsearch/elasticsearch-ruby/tree/master/elasticsearch-api),
|
61
|
-
which provides a Ruby API for the Elasticsearch RESTful API
|
70
|
+
### Identifying running tasks with X-Opaque-Id
|
62
71
|
|
63
|
-
|
72
|
+
The X-Opaque-Id header allows to track certain calls, or associate certain tasks with the client that started them ([more on the Elasticsearch docs](https://www.elastic.co/guide/en/elasticsearch/reference/master/tasks.html#_identifying_running_tasks)). To use this feature, you need to set an id for `opaque_id` on the client on each request. Example:
|
64
73
|
|
65
74
|
```ruby
|
66
|
-
|
75
|
+
client = Elasticsearch::Client.new
|
76
|
+
client.search(index: 'myindex', q: 'title:test', opaque_id: '123456')
|
77
|
+
```
|
78
|
+
The search request will include the following HTTP Header:
|
79
|
+
```
|
80
|
+
X-Opaque-Id: 123456
|
81
|
+
```
|
67
82
|
|
68
|
-
client
|
83
|
+
You can also set a prefix for X-Opaque-Id when initializing the client. This will be prepended to the id you set before each request if you're using X-Opaque-Id. Example:
|
84
|
+
```ruby
|
85
|
+
client = Elastic::Transport::Client.new(opaque_id_prefix: 'eu-west1_')
|
86
|
+
client.search(index: 'myindex', q: 'title:test', opaque_id: '123456')
|
87
|
+
```
|
88
|
+
The request will include the following HTTP Header:
|
89
|
+
```
|
90
|
+
X-Opaque-Id: eu-west1_123456
|
91
|
+
```
|
69
92
|
|
70
|
-
|
93
|
+
### Api Key Authentication
|
71
94
|
|
72
|
-
|
95
|
+
You can use [**API Key authentication**](https://www.elastic.co/guide/en/elasticsearch/reference/current/security-api-create-api-key.html):
|
73
96
|
|
74
|
-
|
97
|
+
``` ruby
|
98
|
+
Elasticsearch::Client.new(
|
99
|
+
host: host,
|
100
|
+
transport_options: transport_options,
|
101
|
+
api_key: credentials
|
102
|
+
)
|
103
|
+
```
|
75
104
|
|
76
|
-
|
105
|
+
Where credentials is either the base64 encoding of `id` and `api_key` joined by a colon or a hash with the `id` and `api_key`:
|
106
|
+
|
107
|
+
``` ruby
|
108
|
+
Elasticsearch::Client.new(
|
109
|
+
host: host,
|
110
|
+
transport_options: transport_options,
|
111
|
+
api_key: {id: 'my_id', api_key: 'my_api_key'}
|
112
|
+
)
|
77
113
|
```
|
78
114
|
|
115
|
+
## API and Transport
|
116
|
+
|
79
117
|
Please refer to the specific library documentation for details:
|
80
118
|
|
81
119
|
* **Transport**:
|
82
|
-
[[README]](https://github.com/
|
83
|
-
[[Documentation]](
|
120
|
+
[[README]](https://github.com/elastic/elastic-transport-ruby#elastic-transport)
|
121
|
+
[[Documentation]](https://rubydoc.info/github/elastic/elastic-transport-ruby/)
|
84
122
|
|
85
123
|
* **API**:
|
86
|
-
[[README]](https://github.com/
|
87
|
-
[[Documentation]](
|
124
|
+
[[README]](https://github.com/elastic/elasticsearch-ruby/tree/main/elasticsearch-api#elasticsearchapi)
|
125
|
+
[[Documentation]](https://rubydoc.info/gems/elasticsearch-api)
|
88
126
|
|
89
127
|
## License
|
90
128
|
|
91
|
-
This software is licensed under the [Apache 2 license](./LICENSE).
|
129
|
+
This software is licensed under the [Apache 2 license](./LICENSE).
|
data/Rakefile
CHANGED
@@ -15,40 +15,35 @@
|
|
15
15
|
# specific language governing permissions and limitations
|
16
16
|
# under the License.
|
17
17
|
|
18
|
-
require
|
18
|
+
require 'bundler/gem_tasks'
|
19
19
|
|
20
|
-
|
21
|
-
|
20
|
+
task(:default) { system 'rake --tasks' }
|
21
|
+
|
22
|
+
desc 'Run unit tests'
|
23
|
+
task test: 'test:spec'
|
22
24
|
|
23
25
|
# ----- Test tasks ------------------------------------------------------------
|
26
|
+
require 'rspec/core/rake_task'
|
24
27
|
|
25
|
-
require 'rake/testtask'
|
26
28
|
namespace :test do
|
27
|
-
desc
|
29
|
+
desc 'Wait for Elasticsearch to be in a green state'
|
28
30
|
task :wait_for_green do
|
29
31
|
sh '../scripts/wait-cluster.sh'
|
30
32
|
end
|
31
33
|
|
32
|
-
|
33
|
-
|
34
|
-
test.test_files = FileList["test/unit/**/*_test.rb"]
|
35
|
-
test.verbose = false
|
36
|
-
test.warning = false
|
34
|
+
RSpec::Core::RakeTask.new(:integration) do |t|
|
35
|
+
t.pattern = 'spec/integration/**{,/*/**}/*_spec.rb'
|
37
36
|
end
|
38
37
|
|
39
|
-
|
40
|
-
|
41
|
-
test.libs << 'lib' << 'test'
|
42
|
-
test.test_files = FileList["test/integration/**/*_test.rb"]
|
43
|
-
test.verbose = false
|
44
|
-
test.warning = false
|
38
|
+
RSpec::Core::RakeTask.new(:unit) do |t|
|
39
|
+
t.pattern = 'spec/unit/**{,/*/**}/*_spec.rb'
|
45
40
|
end
|
46
41
|
|
47
|
-
|
48
|
-
|
49
|
-
|
42
|
+
desc 'Run unit and integration tests'
|
43
|
+
task :all do
|
44
|
+
Rake::Task['test:unit'].invoke
|
45
|
+
Rake::Task['test:integration'].invoke
|
50
46
|
end
|
51
|
-
|
52
47
|
end
|
53
48
|
|
54
49
|
# ----- Documentation tasks ---------------------------------------------------
|
data/bin/elastic_ruby_console
CHANGED
@@ -1,30 +1,14 @@
|
|
1
1
|
#!/usr/bin/env ruby
|
2
2
|
|
3
3
|
$LOAD_PATH.unshift(File.expand_path('../../elasticsearch/lib', __dir__))
|
4
|
-
$LOAD_PATH.unshift(File.expand_path('../../elasticsearch-transport/lib', __dir__))
|
5
|
-
$LOAD_PATH.unshift(File.expand_path('../../elasticsearch-dsl/lib', __dir__))
|
6
4
|
$LOAD_PATH.unshift(File.expand_path('../../elasticsearch-api/lib', __dir__))
|
7
|
-
$LOAD_PATH.unshift(File.expand_path('../../elasticsearch
|
8
|
-
$LOAD_PATH.unshift(File.expand_path('../../elasticsearch-extensions/lib', __dir__))
|
5
|
+
$LOAD_PATH.unshift(File.expand_path('../../elasticsearch/lib/elasticsearch/helpers', __dir__))
|
9
6
|
|
10
7
|
require 'elasticsearch'
|
11
|
-
require 'elasticsearch-transport'
|
12
8
|
require 'elasticsearch-api'
|
13
|
-
|
14
|
-
gems_not_loaded = ['elasticsearch-dsl', 'elasticsearch/xpack', 'elasticsearch-extensions'].reject do |gem|
|
15
|
-
begin
|
16
|
-
(require gem) || true
|
17
|
-
rescue LoadError
|
18
|
-
false
|
19
|
-
end
|
20
|
-
end
|
21
|
-
|
22
|
-
unless gems_not_loaded.empty?
|
23
|
-
warn "The following gems were not loaded: [#{gems_not_loaded.join(', ')}]. Please install and require them explicitly."
|
24
|
-
end
|
9
|
+
require 'elasticsearch/helpers/bulk_helper'
|
25
10
|
|
26
11
|
include Elasticsearch
|
27
|
-
include Elasticsearch::DSL if defined?(Elasticsearch::DSL)
|
28
12
|
|
29
13
|
begin
|
30
14
|
require 'pry'
|
data/elasticsearch.gemspec
CHANGED
@@ -22,54 +22,43 @@ require 'elasticsearch/version'
|
|
22
22
|
Gem::Specification.new do |s|
|
23
23
|
s.name = 'elasticsearch'
|
24
24
|
s.version = Elasticsearch::VERSION
|
25
|
-
s.authors = ['
|
26
|
-
s.email = ['
|
25
|
+
s.authors = ['Elastic Client Library Maintainers']
|
26
|
+
s.email = ['client-libs@elastic.co']
|
27
27
|
s.summary = 'Ruby integrations for Elasticsearch'
|
28
|
-
s.homepage = 'https://www.elastic.co/guide/en/elasticsearch/client/ruby-api/
|
28
|
+
s.homepage = 'https://www.elastic.co/guide/en/elasticsearch/client/ruby-api/current/index.html'
|
29
29
|
s.license = 'Apache-2.0'
|
30
30
|
s.metadata = {
|
31
|
-
'homepage_uri' => 'https://www.elastic.co/guide/en/elasticsearch/client/ruby-api/
|
32
|
-
'changelog_uri' => 'https://github.com/elastic/elasticsearch-ruby/blob/
|
33
|
-
'source_code_uri' => 'https://github.com/elastic/elasticsearch-ruby/tree/
|
31
|
+
'homepage_uri' => 'https://www.elastic.co/guide/en/elasticsearch/client/ruby-api/current/index.html',
|
32
|
+
'changelog_uri' => 'https://github.com/elastic/elasticsearch-ruby/blob/main/CHANGELOG.md',
|
33
|
+
'source_code_uri' => 'https://github.com/elastic/elasticsearch-ruby/tree/main',
|
34
34
|
'bug_tracker_uri' => 'https://github.com/elastic/elasticsearch-ruby/issues'
|
35
35
|
}
|
36
|
-
s.files
|
37
|
-
s.executables
|
38
|
-
s.executables
|
39
|
-
s.test_files
|
36
|
+
s.files = `git ls-files`.split($/)
|
37
|
+
s.executables = s.files.grep(%r{^bin/}) { |f| File.basename(f) }
|
38
|
+
s.executables << 'elastic_ruby_console'
|
39
|
+
s.test_files = s.files.grep(%r{^(test|spec|features)/})
|
40
40
|
s.require_paths = ['lib']
|
41
41
|
s.bindir = 'bin'
|
42
42
|
|
43
|
-
s.extra_rdoc_files = [
|
44
|
-
s.rdoc_options = [
|
43
|
+
s.extra_rdoc_files = ['README.md', 'LICENSE.txt']
|
44
|
+
s.rdoc_options = ['--charset=UTF-8']
|
45
45
|
|
46
|
-
s.required_ruby_version = '>= 2.
|
47
|
-
|
48
|
-
s.add_dependency 'elasticsearch-transport', '7.13.3'
|
49
|
-
s.add_dependency 'elasticsearch-api', '7.13.3'
|
46
|
+
s.required_ruby_version = '>= 2.5'
|
50
47
|
|
48
|
+
s.add_dependency 'elastic-transport', '~> 8.3'
|
49
|
+
s.add_dependency 'elasticsearch-api', '8.14.0'
|
50
|
+
|
51
|
+
s.add_development_dependency 'base64'
|
51
52
|
s.add_development_dependency 'bundler'
|
52
|
-
|
53
|
-
s.add_development_dependency 'rake', '~> 13'
|
54
|
-
|
55
|
-
s.add_development_dependency 'elasticsearch-extensions'
|
56
|
-
|
57
|
-
s.add_development_dependency 'ansi'
|
58
|
-
s.add_development_dependency 'shoulda-context'
|
59
|
-
s.add_development_dependency 'mocha'
|
60
|
-
s.add_development_dependency 'yard'
|
53
|
+
s.add_development_dependency 'debug' unless defined?(JRUBY_VERSION)
|
61
54
|
s.add_development_dependency 'pry'
|
62
|
-
|
63
|
-
|
64
|
-
s.add_development_dependency 'minitest'
|
65
|
-
s.add_development_dependency 'minitest-reporters'
|
66
|
-
s.add_development_dependency 'ruby-prof' unless defined?(JRUBY_VERSION) || defined?(Rubinius)
|
55
|
+
s.add_development_dependency 'rake'
|
67
56
|
s.add_development_dependency 'require-prof' unless defined?(JRUBY_VERSION) || defined?(Rubinius)
|
68
|
-
s.add_development_dependency '
|
69
|
-
s.add_development_dependency '
|
70
|
-
s.add_development_dependency '
|
71
|
-
|
72
|
-
s.add_development_dependency '
|
57
|
+
s.add_development_dependency 'rspec'
|
58
|
+
s.add_development_dependency 'ruby-prof' unless defined?(JRUBY_VERSION) || defined?(Rubinius)
|
59
|
+
s.add_development_dependency 'simplecov'
|
60
|
+
s.add_development_dependency 'webmock'
|
61
|
+
s.add_development_dependency 'yard'
|
73
62
|
|
74
63
|
s.description = <<-DESC.gsub(/^ /, '')
|
75
64
|
Ruby integrations for Elasticsearch (client, API, etc.)
|
@@ -0,0 +1,129 @@
|
|
1
|
+
# Licensed to Elasticsearch B.V. under one or more contributor
|
2
|
+
# license agreements. See the NOTICE file distributed with
|
3
|
+
# this work for additional information regarding copyright
|
4
|
+
# ownership. Elasticsearch B.V. licenses this file to you under
|
5
|
+
# the Apache License, Version 2.0 (the "License"); you may
|
6
|
+
# not use this file except in compliance with the License.
|
7
|
+
# You may obtain a copy of the License at
|
8
|
+
#
|
9
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
10
|
+
#
|
11
|
+
# Unless required by applicable law or agreed to in writing,
|
12
|
+
# software distributed under the License is distributed on an
|
13
|
+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
14
|
+
# KIND, either express or implied. See the License for the
|
15
|
+
# specific language governing permissions and limitations
|
16
|
+
# under the License.
|
17
|
+
|
18
|
+
module Elasticsearch
|
19
|
+
module Helpers
|
20
|
+
# Elasticsearch Client Helper for the Bulk API
|
21
|
+
#
|
22
|
+
# @see https://www.elastic.co/guide/en/elasticsearch/reference/master/docs-bulk.html
|
23
|
+
#
|
24
|
+
class BulkHelper
|
25
|
+
attr_accessor :index
|
26
|
+
|
27
|
+
# Create a BulkHelper
|
28
|
+
#
|
29
|
+
# @param [Elasticsearch::Client] client Instance of Elasticsearch client to use.
|
30
|
+
# @param [String] index Index on which to perform the Bulk actions.
|
31
|
+
# @param [Hash] params Parameters to re-use in every bulk call
|
32
|
+
#
|
33
|
+
def initialize(client, index, params = {})
|
34
|
+
@client = client
|
35
|
+
@index = index
|
36
|
+
@params = params
|
37
|
+
end
|
38
|
+
|
39
|
+
# Index documents using the Bulk API.
|
40
|
+
#
|
41
|
+
# @param [Array<Hash>] docs The documents to be indexed.
|
42
|
+
# @param [Hash] params Parameters to use in the bulk ingestion. See the official Elastic documentation for Bulk API for parameters to send to the Bulk API.
|
43
|
+
# @option params [Integer] slice number of documents to send to the Bulk API for eatch batch of ingestion.
|
44
|
+
# @param block [Block] Optional block to run after ingesting a batch of documents.
|
45
|
+
# @yieldparam response [Elasticsearch::Transport::Response] The response object from calling the Bulk API.
|
46
|
+
# @yieldparam ingest_docs [Array<Hash>] The collection of documents sent in the bulk request.
|
47
|
+
#
|
48
|
+
def ingest(docs, params = {}, body = {}, &block)
|
49
|
+
ingest_docs = docs.map { |doc| { index: { _index: @index, data: doc} } }
|
50
|
+
if (slice = params.delete(:slice))
|
51
|
+
ingest_docs.each_slice(slice) do |items|
|
52
|
+
ingest(items.map { |item| item[:index][:data] }, params, &block)
|
53
|
+
end
|
54
|
+
else
|
55
|
+
bulk_request(ingest_docs, params, &block)
|
56
|
+
end
|
57
|
+
end
|
58
|
+
|
59
|
+
# Delete documents using the Bulk API
|
60
|
+
#
|
61
|
+
# @param [Array] ids Array of id's for documents to delete.
|
62
|
+
# @param [Hash] params Parameters to send to bulk delete.
|
63
|
+
#
|
64
|
+
def delete(ids, params = {}, body = {})
|
65
|
+
delete_docs = ids.map { |id| { delete: { _index: @index, _id: id} } }
|
66
|
+
@client.bulk({ body: delete_docs }.merge(params.merge(@params)))
|
67
|
+
end
|
68
|
+
|
69
|
+
# Update documents using the Bulk API
|
70
|
+
#
|
71
|
+
# @param [Array<Hash>] docs (Required) The documents to be updated.
|
72
|
+
# @option params [Integer] slice number of documents to send to the Bulk API for eatch batch of updates.
|
73
|
+
# @param block [Block] Optional block to run after ingesting a batch of documents.
|
74
|
+
#
|
75
|
+
# @yieldparam response [Elasticsearch::Transport::Response] The response object from calling the Bulk API.
|
76
|
+
# @yieldparam ingest_docs [Array<Hash>] The collection of documents sent in the bulk request.
|
77
|
+
#
|
78
|
+
def update(docs, params = {}, body = {}, &block)
|
79
|
+
ingest_docs = docs.map do |doc|
|
80
|
+
{ update: { _index: @index, _id: doc.delete('id'), data: { doc: doc } } }
|
81
|
+
end
|
82
|
+
if (slice = params.delete(:slice))
|
83
|
+
ingest_docs.each_slice(slice) { |items| update(items, params, &block) }
|
84
|
+
else
|
85
|
+
bulk_request(ingest_docs, params, &block)
|
86
|
+
end
|
87
|
+
end
|
88
|
+
|
89
|
+
# Ingest data directly from a JSON file
|
90
|
+
#
|
91
|
+
# @param [String] file (Required) The file path.
|
92
|
+
# @param [Hash] params Parameters to use in the bulk ingestion.
|
93
|
+
# @option params [Integer] slice number of documents to send to the Bulk API for eatch batch of updates.
|
94
|
+
# @option params [Array|String] keys If the data needs to be digged from the JSON file, the
|
95
|
+
# keys can be passed in with this parameter to find it.
|
96
|
+
#
|
97
|
+
# E.g.: If the data in the parsed JSON Hash is found in
|
98
|
+
# +json_parsed['data']['items']+, keys would be passed
|
99
|
+
# like this (as an Array):
|
100
|
+
#
|
101
|
+
# +bulk_helper.ingest_json(file, { keys: ['data', 'items'] })+
|
102
|
+
#
|
103
|
+
# or as a String:
|
104
|
+
#
|
105
|
+
# +bulk_helper.ingest_json(file, { keys: 'data, items' })+
|
106
|
+
#
|
107
|
+
# @yieldparam response [Elasticsearch::Transport::Response] The response object from calling the Bulk API.
|
108
|
+
# @yieldparam ingest_docs [Array<Hash>] The collection of documents sent in the bulk request.
|
109
|
+
#
|
110
|
+
def ingest_json(file, params = {}, &block)
|
111
|
+
data = JSON.parse(File.read(file))
|
112
|
+
if (keys = params.delete(:keys))
|
113
|
+
keys = keys.split(',') if keys.is_a?(String)
|
114
|
+
data = data.dig(*keys)
|
115
|
+
end
|
116
|
+
|
117
|
+
ingest(data, params, &block)
|
118
|
+
end
|
119
|
+
|
120
|
+
private
|
121
|
+
|
122
|
+
def bulk_request(ingest_docs, params, &block)
|
123
|
+
response = @client.bulk({ body: ingest_docs }.merge(params.merge(@params)))
|
124
|
+
yield response, ingest_docs if block_given?
|
125
|
+
response
|
126
|
+
end
|
127
|
+
end
|
128
|
+
end
|
129
|
+
end
|
@@ -0,0 +1,72 @@
|
|
1
|
+
# Licensed to Elasticsearch B.V. under one or more contributor
|
2
|
+
# license agreements. See the NOTICE file distributed with
|
3
|
+
# this work for additional information regarding copyright
|
4
|
+
# ownership. Elasticsearch B.V. licenses this file to you under
|
5
|
+
# the Apache License, Version 2.0 (the "License"); you may
|
6
|
+
# not use this file except in compliance with the License.
|
7
|
+
# You may obtain a copy of the License at
|
8
|
+
#
|
9
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
10
|
+
#
|
11
|
+
# Unless required by applicable law or agreed to in writing,
|
12
|
+
# software distributed under the License is distributed on an
|
13
|
+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
14
|
+
# KIND, either express or implied. See the License for the
|
15
|
+
# specific language governing permissions and limitations
|
16
|
+
# under the License.
|
17
|
+
|
18
|
+
module Elasticsearch
|
19
|
+
module Helpers
|
20
|
+
# Elasticsearch Client Helper for the ES|QL API
|
21
|
+
#
|
22
|
+
# @see https://www.elastic.co/guide/en/elasticsearch/reference/current/esql-query-api.html
|
23
|
+
#
|
24
|
+
module ESQLHelper
|
25
|
+
# Query helper for ES|QL
|
26
|
+
#
|
27
|
+
# By default, the `esql.query` API returns a Hash response with the following keys:
|
28
|
+
#
|
29
|
+
# * `columns` with the value being an Array of `{ name: type }` Hashes for each column.
|
30
|
+
#
|
31
|
+
# * `values` with the value being an Array of Arrays with the values for each row.
|
32
|
+
#
|
33
|
+
# This helper function returns an Array of hashes with the columns as keys and the respective
|
34
|
+
# values: `{ column['name'] => value }`.
|
35
|
+
#
|
36
|
+
# @param client [Elasticsearch::Client] an instance of the Client to use for the query.
|
37
|
+
# @param query [Hash, String] The query to be passed to the ES|QL query API.
|
38
|
+
# @param params [Hash] options to pass to the ES|QL query API.
|
39
|
+
# @param parser [Hash] Hash of column name keys and Proc values to transform the value of
|
40
|
+
# a given column.
|
41
|
+
# @example Using the ES|QL helper
|
42
|
+
# require 'elasticsearch/helpers/esql_helper'
|
43
|
+
# query = <<~ESQL
|
44
|
+
# FROM sample_data
|
45
|
+
# | EVAL duration_ms = ROUND(event.duration / 1000000.0, 1)
|
46
|
+
# ESQL
|
47
|
+
# response = Elasticsearch::Helpers::ESQLHelper.query(client, query)
|
48
|
+
#
|
49
|
+
# @example Using the ES|QL helper with a parser
|
50
|
+
# response = Elasticsearch::Helpers::ESQLHelper.query(
|
51
|
+
# client,
|
52
|
+
# query,
|
53
|
+
# parser: { '@timestamp' => Proc.new { |t| DateTime.parse(t) } }
|
54
|
+
# )
|
55
|
+
#
|
56
|
+
# @see https://www.elastic.co/guide/en/elasticsearch/client/ruby-api/current/Helpers.html#_esql_helper
|
57
|
+
#
|
58
|
+
def self.query(client, query, params = {}, parser: {})
|
59
|
+
response = client.esql.query({ body: { query: query }, format: 'json' }.merge(params))
|
60
|
+
|
61
|
+
columns = response['columns']
|
62
|
+
response['values'].map do |value|
|
63
|
+
(value.length - 1).downto(0).map do |index|
|
64
|
+
key = columns[index]['name']
|
65
|
+
value[index] = parser[key].call(value[index]) if value[index] && parser[key]
|
66
|
+
{ key => value[index] }
|
67
|
+
end.reduce({}, :merge)
|
68
|
+
end
|
69
|
+
end
|
70
|
+
end
|
71
|
+
end
|
72
|
+
end
|
@@ -0,0 +1,95 @@
|
|
1
|
+
# Licensed to Elasticsearch B.V. under one or more contributor
|
2
|
+
# license agreements. See the NOTICE file distributed with
|
3
|
+
# this work for additional information regarding copyright
|
4
|
+
# ownership. Elasticsearch B.V. licenses this file to you under
|
5
|
+
# the Apache License, Version 2.0 (the "License"); you may
|
6
|
+
# not use this file except in compliance with the License.
|
7
|
+
# You may obtain a copy of the License at
|
8
|
+
#
|
9
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
10
|
+
#
|
11
|
+
# Unless required by applicable law or agreed to in writing,
|
12
|
+
# software distributed under the License is distributed on an
|
13
|
+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
14
|
+
# KIND, either express or implied. See the License for the
|
15
|
+
# specific language governing permissions and limitations
|
16
|
+
# under the License.
|
17
|
+
|
18
|
+
module Elasticsearch
|
19
|
+
module Helpers
|
20
|
+
# Elasticsearch Client Helper for the Scroll API
|
21
|
+
#
|
22
|
+
# @see https://www.elastic.co/guide/en/elasticsearch/reference/current/scroll-api.html
|
23
|
+
#
|
24
|
+
class ScrollHelper
|
25
|
+
include Enumerable
|
26
|
+
|
27
|
+
# Create a ScrollHelper
|
28
|
+
#
|
29
|
+
# @param [Elasticsearch::Client] client (Required) Instance of Elasticsearch client to use.
|
30
|
+
# @param [String] index (Required) Index on which to perform the Bulk actions.
|
31
|
+
# @param [Hash] body Body parameters to re-use in every scroll request
|
32
|
+
# @param [Time] scroll Specify how long a consistent view of the index should be maintained for scrolled search
|
33
|
+
#
|
34
|
+
def initialize(client, index, body, scroll = '1m')
|
35
|
+
@index = index
|
36
|
+
@client = client
|
37
|
+
@scroll = scroll
|
38
|
+
@body = body
|
39
|
+
end
|
40
|
+
|
41
|
+
# Implementation of +each+ for Enumerable module inclusion
|
42
|
+
#
|
43
|
+
# @yieldparam document [Hash] yields a document found in the search hits.
|
44
|
+
#
|
45
|
+
def each(&block)
|
46
|
+
@docs = []
|
47
|
+
@scroll_id = nil
|
48
|
+
refresh_docs
|
49
|
+
for doc in @docs do
|
50
|
+
refresh_docs
|
51
|
+
yield doc
|
52
|
+
end
|
53
|
+
clear
|
54
|
+
end
|
55
|
+
|
56
|
+
# Results from a scroll.
|
57
|
+
# Can be called repeatedly (e.g. in a loop) to get the scroll pages.
|
58
|
+
#
|
59
|
+
def results
|
60
|
+
if @scroll_id
|
61
|
+
scroll_request
|
62
|
+
else
|
63
|
+
initial_search
|
64
|
+
end
|
65
|
+
rescue StandardError => e
|
66
|
+
raise e
|
67
|
+
end
|
68
|
+
|
69
|
+
# Clear Scroll and resets inner documents collection
|
70
|
+
#
|
71
|
+
def clear
|
72
|
+
@client.clear_scroll(body: { scroll_id: @scroll_id }) if @scroll_id
|
73
|
+
@docs = []
|
74
|
+
end
|
75
|
+
|
76
|
+
private
|
77
|
+
|
78
|
+
def refresh_docs
|
79
|
+
@docs ||= []
|
80
|
+
@docs << results
|
81
|
+
@docs.flatten!
|
82
|
+
end
|
83
|
+
|
84
|
+
def initial_search
|
85
|
+
response = @client.search(index: @index, scroll: @scroll, body: @body)
|
86
|
+
@scroll_id = response['_scroll_id']
|
87
|
+
response['hits']['hits']
|
88
|
+
end
|
89
|
+
|
90
|
+
def scroll_request
|
91
|
+
@client.scroll(body: {scroll: @scroll, scroll_id: @scroll_id})['hits']['hits']
|
92
|
+
end
|
93
|
+
end
|
94
|
+
end
|
95
|
+
end
|