logstash-output-elasticsearch 10.7.3-java → 10.8.0-java
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +4 -0
- data/lib/logstash/outputs/elasticsearch.rb +300 -165
- data/lib/logstash/outputs/elasticsearch/http_client.rb +1 -0
- data/lib/logstash/outputs/elasticsearch/http_client/pool.rb +13 -28
- data/lib/logstash/outputs/elasticsearch/http_client_builder.rb +1 -0
- data/lib/logstash/outputs/elasticsearch/ilm.rb +9 -5
- data/lib/logstash/outputs/elasticsearch/license_checker.rb +47 -0
- data/lib/logstash/plugin_mixins/elasticsearch/api_configs.rb +163 -0
- data/lib/logstash/{outputs → plugin_mixins}/elasticsearch/common.rb +40 -167
- data/lib/logstash/plugin_mixins/elasticsearch/noop_license_checker.rb +9 -0
- data/logstash-output-elasticsearch.gemspec +1 -1
- data/spec/unit/outputs/elasticsearch/http_client/pool_spec.rb +45 -5
- data/spec/unit/outputs/license_check_spec.rb +41 -0
- metadata +8 -4
- data/lib/logstash/outputs/elasticsearch/common_configs.rb +0 -167
@@ -15,8 +15,8 @@ describe LogStash::Outputs::ElasticSearch::HttpClient::Pool do
|
|
15
15
|
|
16
16
|
let(:manticore_double) { double("manticore a") }
|
17
17
|
before(:each) do
|
18
|
+
stub_const('LogStash::OSS', oss)
|
18
19
|
|
19
|
-
allow(::LogStash::Outputs::ElasticSearch).to receive(:oss?).and_return(oss)
|
20
20
|
response_double = double("manticore response").as_null_object
|
21
21
|
# Allow healtchecks
|
22
22
|
allow(manticore_double).to receive(:head).with(any_args).and_return(response_double)
|
@@ -26,8 +26,8 @@ describe LogStash::Outputs::ElasticSearch::HttpClient::Pool do
|
|
26
26
|
allow(::Manticore::Client).to receive(:new).and_return(manticore_double)
|
27
27
|
|
28
28
|
allow(subject).to receive(:get_es_version).with(any_args).and_return(*es_node_versions)
|
29
|
-
allow(subject).to receive(:oss?).and_return(oss)
|
30
|
-
allow(subject).to receive(:valid_es_license?).and_return(valid_license)
|
29
|
+
allow(subject.license_checker).to receive(:oss?).and_return(oss)
|
30
|
+
allow(subject.license_checker).to receive(:valid_es_license?).and_return(valid_license)
|
31
31
|
end
|
32
32
|
|
33
33
|
after do
|
@@ -245,27 +245,67 @@ describe LogStash::Outputs::ElasticSearch::HttpClient::Pool do
|
|
245
245
|
before(:each) do
|
246
246
|
allow(subject).to receive(:health_check_request)
|
247
247
|
end
|
248
|
+
|
249
|
+
let(:options) do
|
250
|
+
super.merge(:license_checker => license_checker)
|
251
|
+
end
|
252
|
+
|
253
|
+
context 'when LicenseChecker#acceptable_license? returns false' do
|
254
|
+
let(:license_checker) { double('LicenseChecker', :appropriate_license? => false) }
|
255
|
+
|
256
|
+
it 'does not mark the URL as active' do
|
257
|
+
subject.update_initial_urls
|
258
|
+
expect(subject.alive_urls_count).to eq(0)
|
259
|
+
end
|
260
|
+
end
|
261
|
+
|
262
|
+
context 'when LicenseChecker#acceptable_license? returns true' do
|
263
|
+
let(:license_checker) { double('LicenseChecker', :appropriate_license? => true) }
|
264
|
+
|
265
|
+
it 'marks the URL as active' do
|
266
|
+
subject.update_initial_urls
|
267
|
+
expect(subject.alive_urls_count).to eq(1)
|
268
|
+
end
|
269
|
+
end
|
270
|
+
end
|
271
|
+
|
272
|
+
# TODO: extract to ElasticSearchOutputLicenseChecker unit spec
|
273
|
+
describe "license checking with ElasticSearchOutputLicenseChecker" do
|
274
|
+
let(:options) do
|
275
|
+
super().merge(:license_checker => LogStash::Outputs::ElasticSearch::LicenseChecker.new(logger))
|
276
|
+
end
|
277
|
+
|
278
|
+
before(:each) do
|
279
|
+
allow(subject).to receive(:health_check_request)
|
280
|
+
end
|
281
|
+
|
248
282
|
context "when using default logstash distribution" do
|
249
283
|
let(:oss) { false }
|
284
|
+
|
250
285
|
context "if ES doesn't return a valid license" do
|
251
286
|
let(:valid_license) { false }
|
287
|
+
|
252
288
|
it "marks the url as active" do
|
253
289
|
subject.update_initial_urls
|
254
290
|
expect(subject.alive_urls_count).to eq(1)
|
255
291
|
end
|
292
|
+
|
256
293
|
it "logs a warning" do
|
257
|
-
expect(subject).to receive(:log_license_deprecation_warn).once
|
294
|
+
expect(subject.license_checker).to receive(:log_license_deprecation_warn).once
|
258
295
|
subject.update_initial_urls
|
259
296
|
end
|
260
297
|
end
|
298
|
+
|
261
299
|
context "if ES returns a valid license" do
|
262
300
|
let(:valid_license) { true }
|
301
|
+
|
263
302
|
it "marks the url as active" do
|
264
303
|
subject.update_initial_urls
|
265
304
|
expect(subject.alive_urls_count).to eq(1)
|
266
305
|
end
|
306
|
+
|
267
307
|
it "does not log a warning" do
|
268
|
-
expect(subject).to_not receive(:log_license_deprecation_warn)
|
308
|
+
expect(subject.license_checker).to_not receive(:log_license_deprecation_warn)
|
269
309
|
subject.update_initial_urls
|
270
310
|
end
|
271
311
|
end
|
@@ -0,0 +1,41 @@
|
|
1
|
+
require "logstash/devutils/rspec/spec_helper"
|
2
|
+
require "logstash/outputs/elasticsearch/http_client"
|
3
|
+
require "logstash/outputs/elasticsearch/license_checker"
|
4
|
+
|
5
|
+
describe LogStash::Outputs::ElasticSearch::LicenseChecker do
|
6
|
+
|
7
|
+
# Note that the actual license checking logic is spec'ed in pool_spec.rb
|
8
|
+
|
9
|
+
context "LicenseChecker API required by Pool class" do
|
10
|
+
subject { described_class }
|
11
|
+
|
12
|
+
it "defines the appropriate_license? methods" do
|
13
|
+
expect(subject.instance_methods).to include(:appropriate_license?)
|
14
|
+
end
|
15
|
+
end
|
16
|
+
|
17
|
+
context "LicenseChecker API required by Pool specs" do
|
18
|
+
subject { described_class }
|
19
|
+
|
20
|
+
it "defines the oss? method" do
|
21
|
+
expect(subject.instance_methods).to include(:oss?)
|
22
|
+
end
|
23
|
+
|
24
|
+
it "defines the valid_es_license? method" do
|
25
|
+
expect(subject.instance_methods).to include(:valid_es_license?)
|
26
|
+
end
|
27
|
+
|
28
|
+
it "defines the log_license_deprecation_warn method" do
|
29
|
+
expect(subject.instance_methods).to include(:log_license_deprecation_warn)
|
30
|
+
end
|
31
|
+
end
|
32
|
+
|
33
|
+
context "Pool class API required by the LicenseChecker" do
|
34
|
+
subject { LogStash::Outputs::ElasticSearch::HttpClient::Pool }
|
35
|
+
|
36
|
+
it "contains the get_license method" do
|
37
|
+
expect(LogStash::Outputs::ElasticSearch::HttpClient::Pool.instance_methods).to include(:get_license)
|
38
|
+
end
|
39
|
+
end
|
40
|
+
end
|
41
|
+
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: logstash-output-elasticsearch
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 10.
|
4
|
+
version: 10.8.0
|
5
5
|
platform: java
|
6
6
|
authors:
|
7
7
|
- Elastic
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2020-
|
11
|
+
date: 2020-12-02 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
requirement: !ruby/object:Gem::Requirement
|
@@ -170,14 +170,13 @@ files:
|
|
170
170
|
- README.md
|
171
171
|
- docs/index.asciidoc
|
172
172
|
- lib/logstash/outputs/elasticsearch.rb
|
173
|
-
- lib/logstash/outputs/elasticsearch/common.rb
|
174
|
-
- lib/logstash/outputs/elasticsearch/common_configs.rb
|
175
173
|
- lib/logstash/outputs/elasticsearch/default-ilm-policy.json
|
176
174
|
- lib/logstash/outputs/elasticsearch/http_client.rb
|
177
175
|
- lib/logstash/outputs/elasticsearch/http_client/manticore_adapter.rb
|
178
176
|
- lib/logstash/outputs/elasticsearch/http_client/pool.rb
|
179
177
|
- lib/logstash/outputs/elasticsearch/http_client_builder.rb
|
180
178
|
- lib/logstash/outputs/elasticsearch/ilm.rb
|
179
|
+
- lib/logstash/outputs/elasticsearch/license_checker.rb
|
181
180
|
- lib/logstash/outputs/elasticsearch/template_manager.rb
|
182
181
|
- lib/logstash/outputs/elasticsearch/templates/ecs-disabled/elasticsearch-2x.json
|
183
182
|
- lib/logstash/outputs/elasticsearch/templates/ecs-disabled/elasticsearch-5x.json
|
@@ -186,6 +185,9 @@ files:
|
|
186
185
|
- lib/logstash/outputs/elasticsearch/templates/ecs-disabled/elasticsearch-8x.json
|
187
186
|
- lib/logstash/outputs/elasticsearch/templates/ecs-v1/elasticsearch-6x.json
|
188
187
|
- lib/logstash/outputs/elasticsearch/templates/ecs-v1/elasticsearch-7x.json
|
188
|
+
- lib/logstash/plugin_mixins/elasticsearch/api_configs.rb
|
189
|
+
- lib/logstash/plugin_mixins/elasticsearch/common.rb
|
190
|
+
- lib/logstash/plugin_mixins/elasticsearch/noop_license_checker.rb
|
189
191
|
- logstash-output-elasticsearch.gemspec
|
190
192
|
- spec/es_spec_helper.rb
|
191
193
|
- spec/fixtures/_nodes/2x_1x.json
|
@@ -238,6 +240,7 @@ files:
|
|
238
240
|
- spec/unit/outputs/elasticsearch_spec.rb
|
239
241
|
- spec/unit/outputs/elasticsearch_ssl_spec.rb
|
240
242
|
- spec/unit/outputs/error_whitelist_spec.rb
|
243
|
+
- spec/unit/outputs/license_check_spec.rb
|
241
244
|
homepage: http://logstash.net/
|
242
245
|
licenses:
|
243
246
|
- apache-2.0
|
@@ -316,3 +319,4 @@ test_files:
|
|
316
319
|
- spec/unit/outputs/elasticsearch_spec.rb
|
317
320
|
- spec/unit/outputs/elasticsearch_ssl_spec.rb
|
318
321
|
- spec/unit/outputs/error_whitelist_spec.rb
|
322
|
+
- spec/unit/outputs/license_check_spec.rb
|
@@ -1,167 +0,0 @@
|
|
1
|
-
require 'forwardable' # Needed for logstash core SafeURI. We need to patch this in core: https://github.com/elastic/logstash/pull/5978
|
2
|
-
|
3
|
-
module LogStash; module Outputs; class ElasticSearch
|
4
|
-
module CommonConfigs
|
5
|
-
|
6
|
-
DEFAULT_INDEX_NAME = "logstash-%{+yyyy.MM.dd}"
|
7
|
-
DEFAULT_POLICY = "logstash-policy"
|
8
|
-
DEFAULT_ROLLOVER_ALIAS = 'logstash'
|
9
|
-
|
10
|
-
DEFAULT_HOST = ::LogStash::Util::SafeURI.new("//127.0.0.1")
|
11
|
-
|
12
|
-
def self.included(mod)
|
13
|
-
# The index to write events to. This can be dynamic using the `%{foo}` syntax.
|
14
|
-
# The default value will partition your indices by day so you can more easily
|
15
|
-
# delete old data or only search specific date ranges.
|
16
|
-
# Indexes may not contain uppercase characters.
|
17
|
-
# For weekly indexes ISO 8601 format is recommended, eg. logstash-%{+xxxx.ww}.
|
18
|
-
# LS uses Joda to format the index pattern from event timestamp.
|
19
|
-
# Joda formats are defined http://www.joda.org/joda-time/apidocs/org/joda/time/format/DateTimeFormat.html[here].
|
20
|
-
mod.config :index, :validate => :string
|
21
|
-
|
22
|
-
mod.config :document_type,
|
23
|
-
:validate => :string,
|
24
|
-
:deprecated => "Document types are being deprecated in Elasticsearch 6.0, and removed entirely in 7.0. You should avoid this feature"
|
25
|
-
|
26
|
-
# From Logstash 1.3 onwards, a template is applied to Elasticsearch during
|
27
|
-
# Logstash's startup if one with the name `template_name` does not already exist.
|
28
|
-
# By default, the contents of this template is the default template for
|
29
|
-
# `logstash-%{+YYYY.MM.dd}` which always matches indices based on the pattern
|
30
|
-
# `logstash-*`. Should you require support for other index names, or would like
|
31
|
-
# to change the mappings in the template in general, a custom template can be
|
32
|
-
# specified by setting `template` to the path of a template file.
|
33
|
-
#
|
34
|
-
# Setting `manage_template` to false disables this feature. If you require more
|
35
|
-
# control over template creation, (e.g. creating indices dynamically based on
|
36
|
-
# field names) you should set `manage_template` to false and use the REST
|
37
|
-
# API to apply your templates manually.
|
38
|
-
mod.config :manage_template, :validate => :boolean, :default => true
|
39
|
-
|
40
|
-
# This configuration option defines how the template is named inside Elasticsearch.
|
41
|
-
# Note that if you have used the template management features and subsequently
|
42
|
-
# change this, you will need to prune the old template manually, e.g.
|
43
|
-
#
|
44
|
-
# `curl -XDELETE <http://localhost:9200/_template/OldTemplateName?pretty>`
|
45
|
-
#
|
46
|
-
# where `OldTemplateName` is whatever the former setting was.
|
47
|
-
mod.config :template_name, :validate => :string
|
48
|
-
|
49
|
-
# You can set the path to your own template here, if you so desire.
|
50
|
-
# If not set, the included template will be used.
|
51
|
-
mod.config :template, :validate => :path
|
52
|
-
|
53
|
-
# The template_overwrite option will always overwrite the indicated template
|
54
|
-
# in Elasticsearch with either the one indicated by template or the included one.
|
55
|
-
# This option is set to false by default. If you always want to stay up to date
|
56
|
-
# with the template provided by Logstash, this option could be very useful to you.
|
57
|
-
# Likewise, if you have your own template file managed by puppet, for example, and
|
58
|
-
# you wanted to be able to update it regularly, this option could help there as well.
|
59
|
-
#
|
60
|
-
# Please note that if you are using your own customized version of the Logstash
|
61
|
-
# template (logstash), setting this to true will make Logstash to overwrite
|
62
|
-
# the "logstash" template (i.e. removing all customized settings)
|
63
|
-
mod.config :template_overwrite, :validate => :boolean, :default => false
|
64
|
-
|
65
|
-
# The document ID for the index. Useful for overwriting existing entries in
|
66
|
-
# Elasticsearch with the same ID.
|
67
|
-
mod.config :document_id, :validate => :string
|
68
|
-
|
69
|
-
# The version to use for indexing. Use sprintf syntax like `%{my_version}` to use a field value here.
|
70
|
-
# See https://www.elastic.co/blog/elasticsearch-versioning-support.
|
71
|
-
mod.config :version, :validate => :string
|
72
|
-
|
73
|
-
# The version_type to use for indexing.
|
74
|
-
# See https://www.elastic.co/blog/elasticsearch-versioning-support.
|
75
|
-
# See also https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-index_.html#_version_types
|
76
|
-
mod.config :version_type, :validate => ["internal", 'external', "external_gt", "external_gte", "force"]
|
77
|
-
|
78
|
-
# A routing override to be applied to all processed events.
|
79
|
-
# This can be dynamic using the `%{foo}` syntax.
|
80
|
-
mod.config :routing, :validate => :string
|
81
|
-
|
82
|
-
# For child documents, ID of the associated parent.
|
83
|
-
# This can be dynamic using the `%{foo}` syntax.
|
84
|
-
mod.config :parent, :validate => :string, :default => nil
|
85
|
-
|
86
|
-
# For child documents, name of the join field
|
87
|
-
mod.config :join_field, :validate => :string, :default => nil
|
88
|
-
|
89
|
-
# Sets the host(s) of the remote instance. If given an array it will load balance requests across the hosts specified in the `hosts` parameter.
|
90
|
-
# Remember the `http` protocol uses the http://www.elastic.co/guide/en/elasticsearch/reference/current/modules-http.html#modules-http[http] address (eg. 9200, not 9300).
|
91
|
-
# `"127.0.0.1"`
|
92
|
-
# `["127.0.0.1:9200","127.0.0.2:9200"]`
|
93
|
-
# `["http://127.0.0.1"]`
|
94
|
-
# `["https://127.0.0.1:9200"]`
|
95
|
-
# `["https://127.0.0.1:9200/mypath"]` (If using a proxy on a subpath)
|
96
|
-
# It is important to exclude http://www.elastic.co/guide/en/elasticsearch/reference/current/modules-node.html[dedicated master nodes] from the `hosts` list
|
97
|
-
# to prevent LS from sending bulk requests to the master nodes. So this parameter should only reference either data or client nodes in Elasticsearch.
|
98
|
-
#
|
99
|
-
# Any special characters present in the URLs here MUST be URL escaped! This means `#` should be put in as `%23` for instance.
|
100
|
-
mod.config :hosts, :validate => :uri, :default => [ DEFAULT_HOST ], :list => true
|
101
|
-
|
102
|
-
# Cloud ID, from the Elastic Cloud web console. If set `hosts` should not be used.
|
103
|
-
#
|
104
|
-
# For more details, check out the https://www.elastic.co/guide/en/logstash/current/connecting-to-cloud.html#_cloud_id[cloud documentation]
|
105
|
-
mod.config :cloud_id, :validate => :string
|
106
|
-
|
107
|
-
# Set upsert content for update mode.s
|
108
|
-
# Create a new document with this parameter as json string if `document_id` doesn't exists
|
109
|
-
mod.config :upsert, :validate => :string, :default => ""
|
110
|
-
|
111
|
-
# Enable `doc_as_upsert` for update mode.
|
112
|
-
# Create a new document with source if `document_id` doesn't exist in Elasticsearch
|
113
|
-
mod.config :doc_as_upsert, :validate => :boolean, :default => false
|
114
|
-
|
115
|
-
# Set script name for scripted update mode
|
116
|
-
mod.config :script, :validate => :string, :default => ""
|
117
|
-
|
118
|
-
# Define the type of script referenced by "script" variable
|
119
|
-
# inline : "script" contains inline script
|
120
|
-
# indexed : "script" contains the name of script directly indexed in elasticsearch
|
121
|
-
# file : "script" contains the name of script stored in elasticseach's config directory
|
122
|
-
mod.config :script_type, :validate => ["inline", 'indexed', "file"], :default => ["inline"]
|
123
|
-
|
124
|
-
# Set the language of the used script. If not set, this defaults to painless in ES 5.0
|
125
|
-
mod.config :script_lang, :validate => :string, :default => "painless"
|
126
|
-
|
127
|
-
# Set variable name passed to script (scripted update)
|
128
|
-
mod.config :script_var_name, :validate => :string, :default => "event"
|
129
|
-
|
130
|
-
# if enabled, script is in charge of creating non-existent document (scripted update)
|
131
|
-
mod.config :scripted_upsert, :validate => :boolean, :default => false
|
132
|
-
|
133
|
-
# Set initial interval in seconds between bulk retries. Doubled on each retry up to `retry_max_interval`
|
134
|
-
mod.config :retry_initial_interval, :validate => :number, :default => 2
|
135
|
-
|
136
|
-
# Set max interval in seconds between bulk retries.
|
137
|
-
mod.config :retry_max_interval, :validate => :number, :default => 64
|
138
|
-
|
139
|
-
# The number of times Elasticsearch should internally retry an update/upserted document
|
140
|
-
# See the https://www.elastic.co/guide/en/elasticsearch/guide/current/partial-updates.html[partial updates]
|
141
|
-
# for more info
|
142
|
-
mod.config :retry_on_conflict, :validate => :number, :default => 1
|
143
|
-
|
144
|
-
# Set which ingest pipeline you wish to execute for an event. You can also use event dependent configuration
|
145
|
-
# here like `pipeline => "%{INGEST_PIPELINE}"`
|
146
|
-
mod.config :pipeline, :validate => :string, :default => nil
|
147
|
-
|
148
|
-
|
149
|
-
# -----
|
150
|
-
# ILM configurations (beta)
|
151
|
-
# -----
|
152
|
-
# Flag for enabling Index Lifecycle Management integration.
|
153
|
-
mod.config :ilm_enabled, :validate => [true, false, 'true', 'false', 'auto'], :default => 'auto'
|
154
|
-
|
155
|
-
# Rollover alias used for indexing data. If rollover alias doesn't exist, Logstash will create it and map it to the relevant index
|
156
|
-
mod.config :ilm_rollover_alias, :validate => :string
|
157
|
-
|
158
|
-
# appends “{now/d}-000001” by default for new index creation, subsequent rollover indices will increment based on this pattern i.e. “000002”
|
159
|
-
# {now/d} is date math, and will insert the appropriate value automatically.
|
160
|
-
mod.config :ilm_pattern, :validate => :string, :default => '{now/d}-000001'
|
161
|
-
|
162
|
-
# ILM policy to use, if undefined the default policy will be used.
|
163
|
-
mod.config :ilm_policy, :validate => :string, :default => DEFAULT_POLICY
|
164
|
-
|
165
|
-
end
|
166
|
-
end
|
167
|
-
end end end
|