logstash-output-opensearch 1.1.0-java → 2.0.0-java
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- checksums.yaml.gz.sig +0 -0
- data/COMPATIBILITY.md +5 -3
- data/CONTRIBUTING.md +26 -0
- data/DEVELOPER_GUIDE.md +9 -0
- data/Gemfile +1 -1
- data/MAINTAINERS.md +3 -0
- data/README.md +90 -3
- data/docs/ecs_compatibility.md +42 -0
- data/lib/logstash/outputs/opensearch/http_client/manticore_adapter.rb +122 -10
- data/lib/logstash/outputs/opensearch/http_client/pool.rb +11 -2
- data/lib/logstash/outputs/opensearch/http_client.rb +22 -25
- data/lib/logstash/outputs/opensearch/http_client_builder.rb +9 -3
- data/lib/logstash/outputs/opensearch/template_manager.rb +6 -5
- data/lib/logstash/outputs/opensearch/templates/ecs-disabled/1x_index.json +66 -0
- data/lib/logstash/outputs/opensearch/templates/ecs-disabled/2x.json +44 -0
- data/lib/logstash/outputs/opensearch/templates/ecs-disabled/2x_index.json +66 -0
- data/lib/logstash/outputs/opensearch/templates/ecs-disabled/7x_index.json +66 -0
- data/lib/logstash/outputs/opensearch/templates/ecs-v1/1x.json +3629 -0
- data/lib/logstash/outputs/opensearch/templates/ecs-v1/1x_index.json +3631 -0
- data/lib/logstash/outputs/opensearch/templates/ecs-v1/2x.json +3629 -0
- data/lib/logstash/outputs/opensearch/templates/ecs-v1/2x_index.json +3631 -0
- data/lib/logstash/outputs/opensearch/templates/ecs-v1/7x.json +3629 -0
- data/lib/logstash/outputs/opensearch/templates/ecs-v1/7x_index.json +3631 -0
- data/lib/logstash/outputs/opensearch/templates/ecs-v8/1x.json +5252 -0
- data/lib/logstash/outputs/opensearch/templates/ecs-v8/1x_index.json +5254 -0
- data/lib/logstash/outputs/opensearch/templates/ecs-v8/2x.json +5252 -0
- data/lib/logstash/outputs/opensearch/templates/ecs-v8/2x_index.json +5254 -0
- data/lib/logstash/outputs/opensearch/templates/ecs-v8/7x.json +5252 -0
- data/lib/logstash/outputs/opensearch/templates/ecs-v8/7x_index.json +5254 -0
- data/lib/logstash/outputs/opensearch.rb +10 -3
- data/lib/logstash/plugin_mixins/opensearch/api_configs.rb +26 -0
- data/logstash-output-opensearch.gemspec +19 -3
- data/spec/integration/outputs/compressed_indexing_spec.rb +10 -4
- data/spec/integration/outputs/create_spec.rb +7 -7
- data/spec/integration/outputs/delete_spec.rb +8 -8
- data/spec/integration/outputs/index_spec.rb +52 -10
- data/spec/integration/outputs/index_version_spec.rb +11 -11
- data/spec/integration/outputs/ingest_pipeline_spec.rb +2 -3
- data/spec/integration/outputs/metrics_spec.rb +0 -2
- data/spec/integration/outputs/no_opensearch_on_startup_spec.rb +0 -1
- data/spec/integration/outputs/painless_update_spec.rb +10 -10
- data/spec/integration/outputs/parent_spec.rb +1 -1
- data/spec/integration/outputs/retry_spec.rb +2 -2
- data/spec/integration/outputs/sniffer_spec.rb +2 -2
- data/spec/integration/outputs/templates_spec.rb +83 -59
- data/spec/integration/outputs/update_spec.rb +14 -14
- data/spec/opensearch_spec_helper.rb +12 -2
- data/spec/unit/outputs/opensearch/http_client/manticore_adapter_spec.rb +86 -3
- data/spec/unit/outputs/opensearch/http_client_spec.rb +26 -5
- data/spec/unit/outputs/opensearch/template_manager_spec.rb +11 -4
- data/spec/unit/outputs/opensearch_spec.rb +18 -2
- data.tar.gz.sig +0 -0
- metadata +74 -9
- metadata.gz.sig +0 -0
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 567e7fbd1e9d9e3ca660d5b40c2bfd12e6f7e9504e65f26beeef04834e5c636a
|
4
|
+
data.tar.gz: d1d6c1bf7eb003d32cea988c5ed7bea367213fd8e344e987a9989bc414e22192
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 1c1ba6f915936dde9d654b5883645bd818cb46fad5c821ec4cf7c87cd250a5bf489957f6f4308aa29ed4e5b0f214b2ae1a4f9195d5d022835610ff8f6e21e3c3
|
7
|
+
data.tar.gz: d585a0bafbe30f63411eaa9a733505a49990f86c092e2a6a0a36170d017ff3bc3cc088dc2cd11e77204d5d921867158c258146c6ce0cad10ee25622c703cd2a6
|
checksums.yaml.gz.sig
CHANGED
Binary file
|
data/COMPATIBILITY.md
CHANGED
@@ -5,23 +5,25 @@
|
|
5
5
|
| logstash-output-opensearch | Logstash OSS|
|
6
6
|
| ------------- | ------------- |
|
7
7
|
| 1.0.0+ | 7.13.2 |
|
8
|
+
| 2.0.0+ | 7.13.2 |
|
8
9
|
|
9
10
|
### Matrix for OpenSearch
|
10
11
|
|
11
12
|
| logstash-output-opensearch | OpenSearch |
|
12
13
|
| ------------- | ------------- |
|
13
|
-
| 1.0.0+ | 1.0.0 |
|
14
|
-
|
14
|
+
| 1.0.0+ | 1.0.0 |
|
15
|
+
| 2.0.0+ | 1.0.0 |
|
15
16
|
|
16
17
|
### Matrix for ODFE
|
17
18
|
|
18
19
|
| logstash-output-opensearch | ODFE |
|
19
20
|
| ------------- | ------------- |
|
20
21
|
| 1.0.0+ | 1.x - 1.13.2 |
|
21
|
-
|
22
|
+
| 2.0.0+ | 1.x - 1.13.2 |
|
22
23
|
|
23
24
|
### Matrix for Elasticsearch OSS
|
24
25
|
|
25
26
|
| logstash-output-opensearch | Elasticsearch OSS|
|
26
27
|
| ------------- | ------------- |
|
27
28
|
| 1.0.0+ | 7.x - 7.10.2 |
|
29
|
+
| 2.0.0+ | 7.x - 7.10.2 |
|
data/CONTRIBUTING.md
CHANGED
@@ -6,6 +6,7 @@
|
|
6
6
|
- [Documentation Changes](#documentation-changes)
|
7
7
|
- [Contributing Code](#contributing-code)
|
8
8
|
- [Developer Certificate of Origin](#developer-certificate-of-origin)
|
9
|
+
- [License Headers](#license-headers)
|
9
10
|
- [Review Process](#review-process)
|
10
11
|
|
11
12
|
## Contributing to logstash-output-opensearch
|
@@ -84,6 +85,31 @@ Signed-off-by: Jane Smith <jane.smith@email.com>
|
|
84
85
|
```
|
85
86
|
You may type this line on your own when writing your commit messages. However, if your user.name and user.email are set in your git configs, you can use `-s` or `– – signoff` to add the `Signed-off-by` line to the end of the commit message.
|
86
87
|
|
88
|
+
## License Headers
|
89
|
+
|
90
|
+
New files in your code contributions should contain the following license header. If you are modifying existing files with license headers, or including new files that already have license headers, do not remove or modify them without guidance.
|
91
|
+
|
92
|
+
### Java
|
93
|
+
|
94
|
+
```
|
95
|
+
/*
|
96
|
+
* Copyright OpenSearch Contributors
|
97
|
+
* SPDX-License-Identifier: Apache-2.0
|
98
|
+
*/
|
99
|
+
```
|
100
|
+
|
101
|
+
### Python
|
102
|
+
```
|
103
|
+
# Copyright OpenSearch Contributors
|
104
|
+
# SPDX-License-Identifier: Apache-2.0
|
105
|
+
```
|
106
|
+
|
107
|
+
### Shell
|
108
|
+
```
|
109
|
+
# Copyright OpenSearch Contributors
|
110
|
+
# SPDX-License-Identifier: Apache-2.0
|
111
|
+
```
|
112
|
+
|
87
113
|
## Review Process
|
88
114
|
|
89
115
|
We deeply appreciate everyone who takes the time to make a contribution. We will review all contributions as quickly as possible. As a reminder, [opening an issue](https://github.com/opensearch-project/logstash-output-opensearch/issues/new/choose) discussing your change before you make it is the best way to smooth the PR process. This will prevent a rejection because someone else is already working on the problem, or because the solution is incompatible with the architectural direction.
|
data/DEVELOPER_GUIDE.md
CHANGED
@@ -8,6 +8,7 @@
|
|
8
8
|
- [Run plugin](#run-plugin-in-logstash)
|
9
9
|
- [Configuration for Logstash Output OpenSearch Plugin](#configuration-for-logstash-output-opensearch-plugin)
|
10
10
|
- [Submitting Changes](#submitting-changes)
|
11
|
+
- [Backports](#backports)
|
11
12
|
|
12
13
|
# Developer Guide
|
13
14
|
|
@@ -206,3 +207,11 @@ Authorization to a secure OpenSearch cluster requires read permission at [index
|
|
206
207
|
## Submitting Changes
|
207
208
|
|
208
209
|
See [CONTRIBUTING](CONTRIBUTING.md).
|
210
|
+
|
211
|
+
## Backports
|
212
|
+
|
213
|
+
The Github workflow in [`backport.yml`](.github/workflows/backport.yml) creates backport PRs automatically when the
|
214
|
+
original PR with an appropriate label `backport <backport-branch-name>` is merged to main with the backport workflow
|
215
|
+
run successfully on the PR. For example, if a PR on main needs to be backported to `1.x` branch, add a label
|
216
|
+
`backport 1.x` to the PR and make sure the backport workflow runs on the PR along with other checks. Once this PR is
|
217
|
+
merged to main, the workflow will create a backport PR to the `1.x` branch.
|
data/Gemfile
CHANGED
@@ -17,4 +17,4 @@ use_logstash_source = ENV["LOGSTASH_SOURCE"] && ENV["LOGSTASH_SOURCE"].to_s == "
|
|
17
17
|
if Dir.exist?(logstash_path) && use_logstash_source
|
18
18
|
gem 'logstash-core', :path => "#{logstash_path}/logstash-core"
|
19
19
|
gem 'logstash-core-plugin-api', :path => "#{logstash_path}/logstash-core-plugin-api"
|
20
|
-
end
|
20
|
+
end
|
data/MAINTAINERS.md
CHANGED
@@ -22,6 +22,9 @@ This document explains who the maintainers are (see below), what they do in this
|
|
22
22
|
| Jack Mazanec | [jmazanec15](https://github.com/jmazanec15) | Amazon |
|
23
23
|
| Vamshi Vijay Nakkirtha | [vamshin](https://github.com/vamshin) | Amazon |
|
24
24
|
| Vijayan Balasubramanian | [VijayanB](https://github.com/VijayanB) | Amazon |
|
25
|
+
| Deep Datta | [deepdatta](https://github.com/deepdatta) | Amazon |
|
26
|
+
| David Venable | [dlvenable](https://github.com/dlvenable) | Amazon |
|
27
|
+
| Shivani Shukla | [sshivanii](https://github.com/sshivanii) | Amazon |
|
25
28
|
|
26
29
|
## Maintainer Responsibilities
|
27
30
|
|
data/README.md
CHANGED
@@ -1,9 +1,10 @@
|
|
1
1
|
[![Build and Test logstash-output-opensearch plugin](https://github.com/opensearch-project/logstash-output-opensearch/actions/workflows/CI.yml/badge.svg)](https://github.com/opensearch-project/logstash-output-opensearch/actions/workflows/CI.yml)
|
2
2
|
![PRs welcome!](https://img.shields.io/badge/PRs-welcome!-success)
|
3
|
-
# Logstash
|
3
|
+
# Logstash Output OpenSearch
|
4
4
|
|
5
5
|
- [Welcome!](#welcome)
|
6
6
|
- [Project Resources](#project-resources)
|
7
|
+
- [Configuration for Logstash Output Opensearch Plugin](#configuration-for-logstash-output-opensearch-plugin)
|
7
8
|
- [Code of Conduct](#code-of-conduct)
|
8
9
|
- [License](#license)
|
9
10
|
- [Copyright](#copyright)
|
@@ -12,10 +13,14 @@
|
|
12
13
|
|
13
14
|
**logstash-output-opensearch** is a community-driven, open source fork logstash-output-elasticsearch licensed under the [Apache v2.0 License](LICENSE). For more information, see [opensearch.org](https://opensearch.org/).
|
14
15
|
|
16
|
+
The logstash-output-opensearch plugin helps to ship events from Logstash to OpenSearch cluster.
|
17
|
+
|
15
18
|
## Project Resources
|
16
19
|
|
17
20
|
* [Project Website](https://opensearch.org/)
|
18
|
-
* [Documentation](https://opensearch.org/)
|
21
|
+
* [Detailed Documentation](https://opensearch.org/docs/latest/clients/logstash/ship-to-opensearch/#opensearch-output-plugin)
|
22
|
+
* [Logstash Overview](https://opensearch.org/docs/clients/logstash/index/)
|
23
|
+
* [Developer Guide](DEVELOPER_GUIDE.md)
|
19
24
|
* Need help? Try [Forums](https://discuss.opendistrocommunity.dev/)
|
20
25
|
* [Project Principles](https://opensearch.org/#principles)
|
21
26
|
* [Contributing to OpenSearch](CONTRIBUTING.md)
|
@@ -24,6 +29,88 @@
|
|
24
29
|
* [Admin Responsibilities](ADMINS.md)
|
25
30
|
* [Security](SECURITY.md)
|
26
31
|
|
32
|
+
## Configuration for Logstash Output Opensearch Plugin
|
33
|
+
|
34
|
+
To run the Logstash Output Opensearch plugin, add following configuration in your logstash.conf file.
|
35
|
+
```
|
36
|
+
output {
|
37
|
+
opensearch {
|
38
|
+
hosts => ["hostname:port"]
|
39
|
+
user => "admin"
|
40
|
+
password => "admin"
|
41
|
+
index => "logstash-logs-%{+YYYY.MM.dd}"
|
42
|
+
}
|
43
|
+
}
|
44
|
+
```
|
45
|
+
|
46
|
+
To run the Logstash Output Opensearch plugin using aws_iam authentication, refer to the sample configuration shown below:
|
47
|
+
```
|
48
|
+
output {
|
49
|
+
opensearch {
|
50
|
+
hosts => ["hostname:port"]
|
51
|
+
auth_type => {
|
52
|
+
type => 'aws_iam'
|
53
|
+
aws_access_key_id => 'ACCESS_KEY'
|
54
|
+
aws_secret_access_key => 'SECRET_KEY'
|
55
|
+
region => 'us-west-2'
|
56
|
+
}
|
57
|
+
index => "logstash-logs-%{+YYYY.MM.dd}"
|
58
|
+
}
|
59
|
+
}
|
60
|
+
```
|
61
|
+
|
62
|
+
In addition to the existing authentication mechanisms, if we want to add new authentication then we will be adding them in the configuration by using auth_type.
|
63
|
+
|
64
|
+
Example Configuration for basic authentication:
|
65
|
+
```
|
66
|
+
output {
|
67
|
+
opensearch {
|
68
|
+
hosts => ["hostname:port"]
|
69
|
+
auth_type => {
|
70
|
+
type => 'basic'
|
71
|
+
user => 'admin'
|
72
|
+
password => 'admin'
|
73
|
+
}
|
74
|
+
index => "logstash-logs-%{+YYYY.MM.dd}"
|
75
|
+
}
|
76
|
+
}
|
77
|
+
```
|
78
|
+
|
79
|
+
To ingest data into a `data stream` through logstash, we need to create the data stream and specify the name of data stream and the `op_type` of `create` in the output configuration. The sample configuration is shown below:
|
80
|
+
|
81
|
+
```yml
|
82
|
+
output {
|
83
|
+
opensearch {
|
84
|
+
hosts => ["https://hostname:port"]
|
85
|
+
auth_type => {
|
86
|
+
type => 'basic'
|
87
|
+
user => 'admin'
|
88
|
+
password => 'admin'
|
89
|
+
}
|
90
|
+
index => "my-data-stream"
|
91
|
+
action => "create"
|
92
|
+
}
|
93
|
+
}
|
94
|
+
```
|
95
|
+
|
96
|
+
Starting in 2.0.0, the aws sdk version is bumped to v3. In order for all other AWS plugins to work together, please remove pre-installed aws plugins and install logstash-integration-aws plugin as follows. See also https://github.com/logstash-plugins/logstash-mixin-aws/issues/38
|
97
|
+
```
|
98
|
+
# Remove existing logstash aws plugins and install logstash-integration-aws to keep sdk dependency the same
|
99
|
+
# https://github.com/logstash-plugins/logstash-mixin-aws/issues/38
|
100
|
+
/usr/share/logstash/bin/logstash-plugin remove logstash-input-s3
|
101
|
+
/usr/share/logstash/bin/logstash-plugin remove logstash-input-sqs
|
102
|
+
/usr/share/logstash/bin/logstash-plugin remove logstash-output-s3
|
103
|
+
/usr/share/logstash/bin/logstash-plugin remove logstash-output-sns
|
104
|
+
/usr/share/logstash/bin/logstash-plugin remove logstash-output-sqs
|
105
|
+
/usr/share/logstash/bin/logstash-plugin remove logstash-output-cloudwatch
|
106
|
+
|
107
|
+
/usr/share/logstash/bin/logstash-plugin install --version 0.1.0.pre logstash-integration-aws
|
108
|
+
bin/logstash-plugin install --version 2.0.0 logstash-output-opensearch
|
109
|
+
```
|
110
|
+
## ECS Compatibility
|
111
|
+
[Elastic Common Schema(ECS)](https://www.elastic.co/guide/en/ecs/current/index.html]) compatibility for V8 was added in 1.3.0. For more details on ECS support refer to this [documentation](docs/ecs_compatibility.md).
|
112
|
+
|
113
|
+
|
27
114
|
## Code of Conduct
|
28
115
|
|
29
116
|
This project has adopted the [Amazon Open Source Code of Conduct](CODE_OF_CONDUCT.md). For more information see the [Code of Conduct FAQ](https://aws.github.io/code-of-conduct-faq), or contact [opensource-codeofconduct@amazon.com](mailto:opensource-codeofconduct@amazon.com) with any additional questions or comments.
|
@@ -34,4 +121,4 @@ This project is licensed under the [Apache v2.0 License](LICENSE).
|
|
34
121
|
|
35
122
|
## Copyright
|
36
123
|
|
37
|
-
Copyright OpenSearch Contributors. See [NOTICE](NOTICE) for details.
|
124
|
+
Copyright OpenSearch Contributors. See [NOTICE](NOTICE) for details.
|
@@ -0,0 +1,42 @@
|
|
1
|
+
# ECS Compatibility Support in logstash-output-opensearch
|
2
|
+
Compatibility for ECS v8 was added in release 1.3.0 of the plugin. This would enable the plugin to work with Logstash 8.x without the need to disable ecs_compatibility.
|
3
|
+
The output plugin doesn't create any events itself, but merely forwards events to OpenSearch that were shaped by plugins preceding it in the pipeline. So, it doesn't play a direct role in making the events ECS compatible.
|
4
|
+
However, the default index templates that the plugin installs into OpenSearch in the ecs_compatibility modes (v1 & v8) ensures that the document fields stored in the indices are ECS compatible. OpenSearch would throw errors for documents that have fields which are ECS incompatible and can't be coerced.
|
5
|
+
|
6
|
+
## ECS index templates used by logstash-output-opensearch 1.3.0
|
7
|
+
* v8 [ECS 8.0.0](https://github.com/elastic/ecs/releases/tag/v8.0.0) [ecs-8.0.0/generated/elasticsearch/legacy/template.json](https://raw.githubusercontent.com/elastic/ecs/v8.0.0/generated/elasticsearch/legacy/template.json)
|
8
|
+
* v1 [ECS 1.9.0](https://github.com/elastic/ecs/releases/tag/v1.9.0) [ecs-1.9.0/generated/elasticsearch/7/template.json](https://raw.githubusercontent.com/elastic/ecs/v1.9.0/generated/elasticsearch/7/template.json)
|
9
|
+
|
10
|
+
## ECS incompatibility
|
11
|
+
Incompatibility can arise for an event when it has fields with the same name as an ECS defined field but a type which can't be coerced into the ECS field.
|
12
|
+
It is Ok to have fields that are not defined in ECS [ref](https://www.elastic.co/guide/en/ecs/current/ecs-faq.html#addl-fields).
|
13
|
+
The dynamic template included in the ECS templates would dynamically map any non-ECS fields.
|
14
|
+
|
15
|
+
### Example
|
16
|
+
[ECS defines the "server"](https://www.elastic.co/guide/en/ecs/current/ecs-server.html) field as an object. But let's say the plugin gets the event below, with a string in the `server` field. It will receive an error from OpenSearch as strings can't be coerced into an object and the ECS incompatible event won't be indexed.
|
17
|
+
```
|
18
|
+
{
|
19
|
+
"@timestamp": "2022-08-22T15:39:18.142175244Z",
|
20
|
+
"@version": "1",
|
21
|
+
"message": "Doc1",
|
22
|
+
"server": "remoteserver.com"
|
23
|
+
}
|
24
|
+
```
|
25
|
+
|
26
|
+
Error received from OpenSearch in the Logstash logs
|
27
|
+
```
|
28
|
+
[2022-08-23T00:01:53,366][WARN ][logstash.outputs.opensearch][main][a36555c6fad3f301db8efff2dfbed768fd85e0b6f4ee35626abe62432f83b95d] Could not index event to OpenSearch. {:status=>400, :action=>["index", {:_id=>nil, :_index=>"ecs-logstash-2022.08.23", :routing=>nil}, {"@timestamp"=>2022-08-22T15:39:18.142175244Z, "@version"=>"1", "server"=>"remoteserver.com", "message"=>"Doc1"}], :response=>{"index"=>{"_index"=>"ecs-logstash-2022.08.23", "_id"=>"CAEUyYIBQM7JQrwxF5NR", "status"=>400, "error"=>{"type"=>"mapper_parsing_exception", "reason"=>"object mapping for [server] tried to parse field [server] as object, but found a concrete value"}}}}
|
29
|
+
```
|
30
|
+
## How to ensure ECS compatibility
|
31
|
+
* The plugins in the pipeline that create the events like the `input` and `codec` plugins should all use ECS defined fields.
|
32
|
+
* Filter plugins like [mutate](https://github.com/logstash-plugins/logstash-filter-mutate/blob/main/docs/index.asciidoc) can be used to map incompatible fields into ECS compatible ones.
|
33
|
+
In the above example the `server` field can be mapped to the `server.domain` field to make it compatible.
|
34
|
+
* You can use your own custom template in the plugin using the `template` and `template_name` configs.
|
35
|
+
[According to this](https://www.elastic.co/guide/en/ecs/current/ecs-faq.html#type-interop) some field types can be changed while staying compatible.
|
36
|
+
|
37
|
+
As a last resort the `ecs_compatibility` config for the logstash-output-opensearch can be set to `disabled`.
|
38
|
+
|
39
|
+
|
40
|
+
_______________
|
41
|
+
## References
|
42
|
+
* [Elastic Common Schema (ECS) Reference](https://www.elastic.co/guide/en/ecs/current/index.html)
|
@@ -7,12 +7,31 @@
|
|
7
7
|
# Modifications Copyright OpenSearch Contributors. See
|
8
8
|
# GitHub history for details.
|
9
9
|
|
10
|
-
require '
|
10
|
+
require 'aws-sdk-core'
|
11
11
|
require 'cgi'
|
12
|
+
require 'manticore'
|
13
|
+
require 'uri'
|
12
14
|
|
13
15
|
module LogStash; module Outputs; class OpenSearch; class HttpClient;
|
16
|
+
AWS_DEFAULT_PORT = 443
|
17
|
+
AWS_DEFAULT_PROFILE = 'default'
|
18
|
+
AWS_DEFAULT_PROFILE_CREDENTIAL_RETRY = 0
|
19
|
+
AWS_DEFAULT_PROFILE_CREDENTIAL_TIMEOUT = 1
|
20
|
+
AWS_DEFAULT_REGION = 'us-east-1'
|
21
|
+
AWS_IAM_AUTH_TYPE = "aws_iam"
|
22
|
+
AWS_SERVICE = 'es'
|
23
|
+
BASIC_AUTH_TYPE = 'basic'
|
14
24
|
DEFAULT_HEADERS = { "content-type" => "application/json" }
|
15
|
-
|
25
|
+
|
26
|
+
AWSIAMCredential = Struct.new(
|
27
|
+
:access_key_id,
|
28
|
+
:secret_access_key,
|
29
|
+
:session_token,
|
30
|
+
:profile,
|
31
|
+
:instance_profile_credentials_retries,
|
32
|
+
:instance_profile_credentials_timeout,
|
33
|
+
:region)
|
34
|
+
|
16
35
|
class ManticoreAdapter
|
17
36
|
attr_reader :manticore, :logger
|
18
37
|
|
@@ -27,14 +46,75 @@ module LogStash; module Outputs; class OpenSearch; class HttpClient;
|
|
27
46
|
options[:cookies] = false
|
28
47
|
|
29
48
|
@client_params = {:headers => DEFAULT_HEADERS.merge(options[:headers] || {})}
|
30
|
-
|
49
|
+
@type = get_auth_type(options) || nil
|
50
|
+
|
51
|
+
if @type == AWS_IAM_AUTH_TYPE
|
52
|
+
aws_iam_auth_initialization(options)
|
53
|
+
elsif @type == BASIC_AUTH_TYPE
|
54
|
+
basic_auth_initialization(options)
|
55
|
+
end
|
56
|
+
|
31
57
|
if options[:proxy]
|
32
58
|
options[:proxy] = manticore_proxy_hash(options[:proxy])
|
33
59
|
end
|
34
|
-
|
60
|
+
|
35
61
|
@manticore = ::Manticore::Client.new(options)
|
36
62
|
end
|
37
|
-
|
63
|
+
|
64
|
+
def get_auth_type(options)
|
65
|
+
if options[:auth_type] != nil
|
66
|
+
options[:auth_type]["type"]
|
67
|
+
end
|
68
|
+
end
|
69
|
+
|
70
|
+
def aws_iam_auth_initialization(options)
|
71
|
+
aws_access_key_id = options[:auth_type]["aws_access_key_id"] || nil
|
72
|
+
aws_secret_access_key = options[:auth_type]["aws_secret_access_key"] || nil
|
73
|
+
session_token = options[:auth_type]["session_token"] || nil
|
74
|
+
profile = options[:auth_type]["profile"] || AWS_DEFAULT_PROFILE
|
75
|
+
instance_cred_retries = options[:auth_type]["instance_profile_credentials_retries"] || AWS_DEFAULT_PROFILE_CREDENTIAL_RETRY
|
76
|
+
instance_cred_timeout = options[:auth_type]["instance_profile_credentials_timeout"] || AWS_DEFAULT_PROFILE_CREDENTIAL_TIMEOUT
|
77
|
+
region = options[:auth_type]["region"] || AWS_DEFAULT_REGION
|
78
|
+
set_aws_region(region)
|
79
|
+
set_service_name(options[:auth_type]["service_name"] || AWS_SERVICE)
|
80
|
+
|
81
|
+
credential_config = AWSIAMCredential.new(aws_access_key_id, aws_secret_access_key, session_token, profile, instance_cred_retries, instance_cred_timeout, region)
|
82
|
+
@credentials = Aws::CredentialProviderChain.new(credential_config).resolve
|
83
|
+
end
|
84
|
+
|
85
|
+
def basic_auth_initialization(options)
|
86
|
+
set_user_password(options)
|
87
|
+
end
|
88
|
+
|
89
|
+
def set_aws_region(region)
|
90
|
+
@region = region
|
91
|
+
end
|
92
|
+
|
93
|
+
def get_aws_region()
|
94
|
+
@region
|
95
|
+
end
|
96
|
+
|
97
|
+
def set_service_name(service_name)
|
98
|
+
@service_name = service_name
|
99
|
+
end
|
100
|
+
|
101
|
+
def get_service_name()
|
102
|
+
@service_name
|
103
|
+
end
|
104
|
+
|
105
|
+
def set_user_password(options)
|
106
|
+
@user = options[:auth_type]["user"]
|
107
|
+
@password = options[:auth_type]["password"]
|
108
|
+
end
|
109
|
+
|
110
|
+
def get_user()
|
111
|
+
@user
|
112
|
+
end
|
113
|
+
|
114
|
+
def get_password()
|
115
|
+
@password
|
116
|
+
end
|
117
|
+
|
38
118
|
# Transform the proxy option to a hash. Manticore's support for non-hash
|
39
119
|
# proxy options is broken. This was fixed in https://github.com/cheald/manticore/commit/34a00cee57a56148629ed0a47c329181e7319af5
|
40
120
|
# but this is not yet released
|
@@ -61,19 +141,28 @@ module LogStash; module Outputs; class OpenSearch; class HttpClient;
|
|
61
141
|
params = (params || {}).merge(@client_params) { |key, oldval, newval|
|
62
142
|
(oldval.is_a?(Hash) && newval.is_a?(Hash)) ? oldval.merge(newval) : newval
|
63
143
|
}
|
144
|
+
|
145
|
+
params[:headers] = params[:headers].clone
|
64
146
|
params[:body] = body if body
|
65
147
|
|
66
148
|
if url.user
|
67
|
-
params[:auth] = {
|
149
|
+
params[:auth] = {
|
68
150
|
:user => CGI.unescape(url.user),
|
69
151
|
# We have to unescape the password here since manticore won't do it
|
70
152
|
# for us unless its part of the URL
|
71
|
-
:password => CGI.unescape(url.password),
|
72
|
-
:eager => true
|
153
|
+
:password => CGI.unescape(url.password),
|
154
|
+
:eager => true
|
73
155
|
}
|
156
|
+
elsif @type == BASIC_AUTH_TYPE
|
157
|
+
add_basic_auth_to_params(params)
|
74
158
|
end
|
75
159
|
|
76
160
|
request_uri = format_url(url, path)
|
161
|
+
|
162
|
+
if @type == AWS_IAM_AUTH_TYPE
|
163
|
+
sign_aws_request(request_uri, path, method, params)
|
164
|
+
end
|
165
|
+
|
77
166
|
request_uri_as_string = remove_double_escaping(request_uri.to_s)
|
78
167
|
resp = @manticore.send(method.downcase, request_uri_as_string, params)
|
79
168
|
|
@@ -92,10 +181,33 @@ module LogStash; module Outputs; class OpenSearch; class HttpClient;
|
|
92
181
|
resp
|
93
182
|
end
|
94
183
|
|
184
|
+
def sign_aws_request(request_uri, path, method, params)
|
185
|
+
url = URI::HTTPS.build({:host=>URI(request_uri.to_s).host, :port=>AWS_DEFAULT_PORT.to_s, :path=>path})
|
186
|
+
request = Seahorse::Client::Http::Request.new(options={:endpoint=>url, :http_method => method.to_s.upcase,
|
187
|
+
:headers => params[:headers],:body => params[:body]})
|
188
|
+
|
189
|
+
aws_signer = Aws::Sigv4::Signer.new(service: @service_name, region: @region, credentials_provider: @credentials)
|
190
|
+
signed_key = aws_signer.sign_request(
|
191
|
+
http_method: request.http_method,
|
192
|
+
url: url,
|
193
|
+
headers: params[:headers],
|
194
|
+
body: params[:body]
|
195
|
+
)
|
196
|
+
params[:headers] = params[:headers].merge(signed_key.headers)
|
197
|
+
end
|
198
|
+
|
199
|
+
def add_basic_auth_to_params(params)
|
200
|
+
params[:auth] = {
|
201
|
+
:user => get_user(),
|
202
|
+
:password => get_password(),
|
203
|
+
:eager => true
|
204
|
+
}
|
205
|
+
end
|
206
|
+
|
95
207
|
# Returned urls from this method should be checked for double escaping.
|
96
208
|
def format_url(url, path_and_query=nil)
|
97
209
|
request_uri = url.clone
|
98
|
-
|
210
|
+
|
99
211
|
# We excise auth info from the URL in case manticore itself tries to stick
|
100
212
|
# sensitive data in a thrown exception or log data
|
101
213
|
request_uri.user = nil
|
@@ -111,7 +223,7 @@ module LogStash; module Outputs; class OpenSearch; class HttpClient;
|
|
111
223
|
new_query_parts = [request_uri.query, parsed_path_and_query.query].select do |part|
|
112
224
|
part && !part.empty? # Skip empty nil and ""
|
113
225
|
end
|
114
|
-
|
226
|
+
|
115
227
|
request_uri.query = new_query_parts.join("&") unless new_query_parts.empty?
|
116
228
|
|
117
229
|
# use `raw_path`` as `path` will unescape any escaped '/' in the path
|
@@ -40,6 +40,7 @@ module LogStash; module Outputs; class OpenSearch; class HttpClient;
|
|
40
40
|
end
|
41
41
|
|
42
42
|
attr_reader :logger, :adapter, :sniffing, :sniffer_delay, :resurrect_delay, :healthcheck_path, :sniffing_path, :bulk_path
|
43
|
+
attr_reader :default_server_major_version
|
43
44
|
|
44
45
|
ROOT_URI_PATH = '/'.freeze
|
45
46
|
|
@@ -68,6 +69,7 @@ module LogStash; module Outputs; class OpenSearch; class HttpClient;
|
|
68
69
|
@resurrect_delay = merged[:resurrect_delay]
|
69
70
|
@sniffing = merged[:sniffing]
|
70
71
|
@sniffer_delay = merged[:sniffer_delay]
|
72
|
+
@default_server_major_version = merged[:default_server_major_version]
|
71
73
|
end
|
72
74
|
|
73
75
|
# Used for all concurrent operations in this class
|
@@ -412,8 +414,15 @@ module LogStash; module Outputs; class OpenSearch; class HttpClient;
|
|
412
414
|
end
|
413
415
|
|
414
416
|
def get_version(url)
|
415
|
-
|
416
|
-
|
417
|
+
response = perform_request_to_url(url, :get, ROOT_URI_PATH)
|
418
|
+
if response.code != 404 && !response.body.empty?
|
419
|
+
return LogStash::Json.load(response.body)["version"]["number"] # e.g. "7.10.0"
|
420
|
+
end
|
421
|
+
if @default_server_major_version.nil?
|
422
|
+
@logger.error("Failed to get version from health_check endpoint and default_server_major_version is not configured.")
|
423
|
+
raise "get_version failed! no default_server_major_version configured."
|
424
|
+
end
|
425
|
+
"#{default_server_major_version}.0.0"
|
417
426
|
end
|
418
427
|
|
419
428
|
def last_version
|
@@ -15,23 +15,9 @@ require 'zlib'
|
|
15
15
|
require 'stringio'
|
16
16
|
|
17
17
|
module LogStash; module Outputs; class OpenSearch;
|
18
|
-
# This is a constant instead of a config option because
|
19
|
-
# there really isn't a good reason to configure it.
|
20
|
-
#
|
21
|
-
# The criteria used are:
|
22
|
-
# 1. We need a number that's less than 100MiB because OpenSearch
|
23
|
-
# won't accept bulks larger than that.
|
24
|
-
# 2. It must be large enough to amortize the connection constant
|
25
|
-
# across multiple requests.
|
26
|
-
# 3. It must be small enough that even if multiple threads hit this size
|
27
|
-
# we won't use a lot of heap.
|
28
|
-
#
|
29
|
-
# We wound up agreeing that a number greater than 10 MiB and less than 100MiB
|
30
|
-
# made sense. We picked one on the lowish side to not use too much heap.
|
31
|
-
TARGET_BULK_BYTES = 20 * 1024 * 1024 # 20MiB
|
32
|
-
|
33
18
|
class HttpClient
|
34
|
-
attr_reader :client, :options, :logger, :pool, :action_count, :recv_count
|
19
|
+
attr_reader :client, :options, :logger, :pool, :action_count, :recv_count, :target_bulk_bytes
|
20
|
+
|
35
21
|
# This is here in case we use DEFAULT_OPTIONS in the future
|
36
22
|
# DEFAULT_OPTIONS = {
|
37
23
|
# :setting => value
|
@@ -41,13 +27,14 @@ module LogStash; module Outputs; class OpenSearch;
|
|
41
27
|
# The `options` is a hash where the following symbol keys have meaning:
|
42
28
|
#
|
43
29
|
# * `:hosts` - array of String. Set a list of hosts to use for communication.
|
44
|
-
# * `:port` - number. set the port to use to communicate with OpenSearch
|
45
30
|
# * `:user` - String. The user to use for authentication.
|
46
31
|
# * `:password` - String. The password to use for authentication.
|
47
32
|
# * `:timeout` - Float. A duration value, in seconds, after which a socket
|
48
33
|
# operation or request will be aborted if not yet successfull
|
34
|
+
# * `:auth_type` - hash of String. It contains the type of authentication
|
35
|
+
# and it's respective credentials
|
49
36
|
# * `:client_settings` - a hash; see below for keys.
|
50
|
-
|
37
|
+
|
51
38
|
# The `client_settings` key is a has that can contain other settings:
|
52
39
|
#
|
53
40
|
# * `:ssl` - Boolean. Enable or disable SSL/TLS.
|
@@ -72,6 +59,8 @@ module LogStash; module Outputs; class OpenSearch;
|
|
72
59
|
# mutex to prevent requests and sniffing to access the
|
73
60
|
# connection pool at the same time
|
74
61
|
@bulk_path = @options[:bulk_path]
|
62
|
+
|
63
|
+
@target_bulk_bytes = @options[:target_bulk_bytes]
|
75
64
|
end
|
76
65
|
|
77
66
|
def build_url_template
|
@@ -104,7 +93,6 @@ module LogStash; module Outputs; class OpenSearch;
|
|
104
93
|
def bulk(actions)
|
105
94
|
@action_count ||= 0
|
106
95
|
@action_count += actions.size
|
107
|
-
|
108
96
|
return if actions.empty?
|
109
97
|
|
110
98
|
bulk_actions = actions.collect do |action, args, source|
|
@@ -131,7 +119,7 @@ module LogStash; module Outputs; class OpenSearch;
|
|
131
119
|
action.map {|line| LogStash::Json.dump(line)}.join("\n") :
|
132
120
|
LogStash::Json.dump(action)
|
133
121
|
as_json << "\n"
|
134
|
-
if (stream_writer.pos + as_json.bytesize) >
|
122
|
+
if (stream_writer.pos + as_json.bytesize) > @target_bulk_bytes && stream_writer.pos > 0
|
135
123
|
stream_writer.flush # ensure writer has sync'd buffers before reporting sizes
|
136
124
|
logger.debug("Sending partial bulk request for batch with one or more actions remaining.",
|
137
125
|
:action_count => batch_actions.size,
|
@@ -324,6 +312,8 @@ module LogStash; module Outputs; class OpenSearch;
|
|
324
312
|
|
325
313
|
adapter_options[:headers] = client_settings[:headers] if client_settings[:headers]
|
326
314
|
|
315
|
+
adapter_options[:auth_type] = options[:auth_type]
|
316
|
+
|
327
317
|
adapter_class = ::LogStash::Outputs::OpenSearch::HttpClient::ManticoreAdapter
|
328
318
|
adapter = adapter_class.new(@logger, adapter_options)
|
329
319
|
end
|
@@ -338,7 +328,8 @@ module LogStash; module Outputs; class OpenSearch;
|
|
338
328
|
:healthcheck_path => options[:healthcheck_path],
|
339
329
|
:resurrect_delay => options[:resurrect_delay],
|
340
330
|
:url_normalizer => self.method(:host_to_url),
|
341
|
-
:metric => options[:metric]
|
331
|
+
:metric => options[:metric],
|
332
|
+
:default_server_major_version => options[:default_server_major_version]
|
342
333
|
}
|
343
334
|
pool_options[:scheme] = self.scheme if self.scheme
|
344
335
|
|
@@ -393,15 +384,21 @@ module LogStash; module Outputs; class OpenSearch;
|
|
393
384
|
end
|
394
385
|
|
395
386
|
def template_put(name, template)
|
396
|
-
path = "
|
387
|
+
path = "/#{template_endpoint}/#{name}"
|
397
388
|
logger.info("Installing OpenSearch template", name: name)
|
398
389
|
@pool.put(path, nil, LogStash::Json.dump(template))
|
399
390
|
end
|
400
391
|
|
392
|
+
def legacy_template?()
|
393
|
+
# TODO: Also check Version and return true for < 7.8 even if :legacy_template=false
|
394
|
+
# Need to figure a way to distinguish between OpenSearch, OpenDistro and other
|
395
|
+
# variants, since they have version numbers in different ranges.
|
396
|
+
client_settings.fetch(:legacy_template, true)
|
397
|
+
end
|
398
|
+
|
401
399
|
def template_endpoint
|
402
|
-
#
|
403
|
-
|
404
|
-
'_template'
|
400
|
+
# https://opensearch.org/docs/opensearch/index-templates/
|
401
|
+
legacy_template?() ? '_template' : '_index_template'
|
405
402
|
end
|
406
403
|
|
407
404
|
# check whether rollover alias already exists
|
@@ -18,7 +18,8 @@ module LogStash; module Outputs; class OpenSearch;
|
|
18
18
|
:pool_max_per_route => params["pool_max_per_route"],
|
19
19
|
:check_connection_timeout => params["validate_after_inactivity"],
|
20
20
|
:http_compression => params["http_compression"],
|
21
|
-
:headers => params["custom_headers"] || {}
|
21
|
+
:headers => params["custom_headers"] || {},
|
22
|
+
:legacy_template => params["legacy_template"]
|
22
23
|
}
|
23
24
|
|
24
25
|
client_settings[:proxy] = params["proxy"] if params["proxy"]
|
@@ -26,7 +27,8 @@ module LogStash; module Outputs; class OpenSearch;
|
|
26
27
|
common_options = {
|
27
28
|
:client_settings => client_settings,
|
28
29
|
:metric => params["metric"],
|
29
|
-
:resurrect_delay => params["resurrect_delay"]
|
30
|
+
:resurrect_delay => params["resurrect_delay"],
|
31
|
+
:default_server_major_version => params["default_server_major_version"]
|
30
32
|
}
|
31
33
|
|
32
34
|
if params["sniffing"]
|
@@ -35,6 +37,7 @@ module LogStash; module Outputs; class OpenSearch;
|
|
35
37
|
end
|
36
38
|
|
37
39
|
common_options[:timeout] = params["timeout"] if params["timeout"]
|
40
|
+
common_options[:target_bulk_bytes] = params["target_bulk_bytes"]
|
38
41
|
|
39
42
|
if params["path"]
|
40
43
|
client_settings[:path] = dedup_slashes("/#{params["path"]}/")
|
@@ -106,7 +109,10 @@ module LogStash; module Outputs; class OpenSearch;
|
|
106
109
|
}
|
107
110
|
common_options.merge! update_options if params["action"] == 'update'
|
108
111
|
|
109
|
-
create_http_client(common_options.merge(:hosts => hosts,
|
112
|
+
create_http_client(common_options.merge(:hosts => hosts,
|
113
|
+
:logger => logger,
|
114
|
+
:auth_type => params["auth_type"]
|
115
|
+
))
|
110
116
|
end
|
111
117
|
|
112
118
|
def self.create_http_client(options)
|