logstash-output-elasticsearch-test 10.3.0-x86_64-linux

Sign up to get free protection for your applications and to get access to all the features.
Files changed (75) hide show
  1. checksums.yaml +7 -0
  2. data/CHANGELOG.md +397 -0
  3. data/CONTRIBUTORS +33 -0
  4. data/Gemfile +15 -0
  5. data/LICENSE +13 -0
  6. data/NOTICE.TXT +5 -0
  7. data/README.md +106 -0
  8. data/docs/index.asciidoc +899 -0
  9. data/lib/logstash/outputs/elasticsearch/common.rb +441 -0
  10. data/lib/logstash/outputs/elasticsearch/common_configs.rb +167 -0
  11. data/lib/logstash/outputs/elasticsearch/default-ilm-policy.json +14 -0
  12. data/lib/logstash/outputs/elasticsearch/elasticsearch-template-es2x.json +95 -0
  13. data/lib/logstash/outputs/elasticsearch/elasticsearch-template-es5x.json +46 -0
  14. data/lib/logstash/outputs/elasticsearch/elasticsearch-template-es6x.json +45 -0
  15. data/lib/logstash/outputs/elasticsearch/elasticsearch-template-es7x.json +44 -0
  16. data/lib/logstash/outputs/elasticsearch/elasticsearch-template-es8x.json +44 -0
  17. data/lib/logstash/outputs/elasticsearch/http_client/manticore_adapter.rb +131 -0
  18. data/lib/logstash/outputs/elasticsearch/http_client/pool.rb +495 -0
  19. data/lib/logstash/outputs/elasticsearch/http_client.rb +432 -0
  20. data/lib/logstash/outputs/elasticsearch/http_client_builder.rb +159 -0
  21. data/lib/logstash/outputs/elasticsearch/ilm.rb +113 -0
  22. data/lib/logstash/outputs/elasticsearch/template_manager.rb +61 -0
  23. data/lib/logstash/outputs/elasticsearch.rb +263 -0
  24. data/logstash-output-elasticsearch.gemspec +33 -0
  25. data/spec/es_spec_helper.rb +189 -0
  26. data/spec/fixtures/_nodes/2x_1x.json +27 -0
  27. data/spec/fixtures/_nodes/5x_6x.json +81 -0
  28. data/spec/fixtures/_nodes/7x.json +92 -0
  29. data/spec/fixtures/htpasswd +2 -0
  30. data/spec/fixtures/nginx_reverse_proxy.conf +22 -0
  31. data/spec/fixtures/scripts/groovy/scripted_update.groovy +2 -0
  32. data/spec/fixtures/scripts/groovy/scripted_update_nested.groovy +2 -0
  33. data/spec/fixtures/scripts/groovy/scripted_upsert.groovy +2 -0
  34. data/spec/fixtures/scripts/painless/scripted_update.painless +2 -0
  35. data/spec/fixtures/scripts/painless/scripted_update_nested.painless +1 -0
  36. data/spec/fixtures/scripts/painless/scripted_upsert.painless +1 -0
  37. data/spec/fixtures/template-with-policy-es6x.json +48 -0
  38. data/spec/fixtures/template-with-policy-es7x.json +45 -0
  39. data/spec/fixtures/test_certs/ca/ca.crt +32 -0
  40. data/spec/fixtures/test_certs/ca/ca.key +51 -0
  41. data/spec/fixtures/test_certs/test.crt +36 -0
  42. data/spec/fixtures/test_certs/test.key +51 -0
  43. data/spec/integration/outputs/compressed_indexing_spec.rb +69 -0
  44. data/spec/integration/outputs/create_spec.rb +67 -0
  45. data/spec/integration/outputs/delete_spec.rb +65 -0
  46. data/spec/integration/outputs/groovy_update_spec.rb +150 -0
  47. data/spec/integration/outputs/ilm_spec.rb +531 -0
  48. data/spec/integration/outputs/index_spec.rb +178 -0
  49. data/spec/integration/outputs/index_version_spec.rb +102 -0
  50. data/spec/integration/outputs/ingest_pipeline_spec.rb +74 -0
  51. data/spec/integration/outputs/metrics_spec.rb +70 -0
  52. data/spec/integration/outputs/no_es_on_startup_spec.rb +58 -0
  53. data/spec/integration/outputs/painless_update_spec.rb +189 -0
  54. data/spec/integration/outputs/parent_spec.rb +102 -0
  55. data/spec/integration/outputs/retry_spec.rb +169 -0
  56. data/spec/integration/outputs/routing_spec.rb +61 -0
  57. data/spec/integration/outputs/sniffer_spec.rb +133 -0
  58. data/spec/integration/outputs/templates_5x_spec.rb +98 -0
  59. data/spec/integration/outputs/templates_spec.rb +98 -0
  60. data/spec/integration/outputs/update_spec.rb +116 -0
  61. data/spec/support/elasticsearch/api/actions/delete_ilm_policy.rb +19 -0
  62. data/spec/support/elasticsearch/api/actions/get_alias.rb +18 -0
  63. data/spec/support/elasticsearch/api/actions/get_ilm_policy.rb +18 -0
  64. data/spec/support/elasticsearch/api/actions/put_alias.rb +24 -0
  65. data/spec/support/elasticsearch/api/actions/put_ilm_policy.rb +25 -0
  66. data/spec/unit/http_client_builder_spec.rb +185 -0
  67. data/spec/unit/outputs/elasticsearch/http_client/manticore_adapter_spec.rb +149 -0
  68. data/spec/unit/outputs/elasticsearch/http_client/pool_spec.rb +274 -0
  69. data/spec/unit/outputs/elasticsearch/http_client_spec.rb +250 -0
  70. data/spec/unit/outputs/elasticsearch/template_manager_spec.rb +25 -0
  71. data/spec/unit/outputs/elasticsearch_proxy_spec.rb +72 -0
  72. data/spec/unit/outputs/elasticsearch_spec.rb +675 -0
  73. data/spec/unit/outputs/elasticsearch_ssl_spec.rb +82 -0
  74. data/spec/unit/outputs/error_whitelist_spec.rb +54 -0
  75. metadata +300 -0
@@ -0,0 +1,61 @@
1
+ module LogStash; module Outputs; class ElasticSearch
2
+ class TemplateManager
3
+ # To be mixed into the elasticsearch plugin base
4
+ def self.install_template(plugin)
5
+ return unless plugin.manage_template
6
+ if plugin.template.nil?
7
+ plugin.logger.info("Using default mapping template")
8
+ else
9
+ plugin.logger.info("Using mapping template from", :path => plugin.template)
10
+ end
11
+
12
+
13
+ template = get_template(plugin.template, plugin.maximum_seen_major_version)
14
+ add_ilm_settings_to_template(plugin, template) if plugin.ilm_in_use?
15
+ plugin.logger.info("Attempting to install template", :manage_template => template)
16
+ install(plugin.client, template_name(plugin), template, plugin.template_overwrite)
17
+ rescue => e
18
+ plugin.logger.error("Failed to install template.", :message => e.message, :class => e.class.name, :backtrace => e.backtrace)
19
+ end
20
+
21
+ private
22
+ def self.get_template(path, es_major_version)
23
+ template_path = path || default_template_path(es_major_version)
24
+ read_template_file(template_path)
25
+ end
26
+
27
+ def self.install(client, template_name, template, template_overwrite)
28
+ client.template_install(template_name, template, template_overwrite)
29
+ end
30
+
31
+ def self.add_ilm_settings_to_template(plugin, template)
32
+ # Overwrite any index patterns, and use the rollover alias. Use 'index_patterns' rather than 'template' for pattern
33
+ # definition - remove any existing definition of 'template'
34
+ template.delete('template') if template.include?('template')
35
+ template['index_patterns'] = "#{plugin.ilm_rollover_alias}-*"
36
+ if template['settings'] && (template['settings']['index.lifecycle.name'] || template['settings']['index.lifecycle.rollover_alias'])
37
+ plugin.logger.info("Overwriting index lifecycle name and rollover alias as ILM is enabled.")
38
+ end
39
+ template['settings'].update({ 'index.lifecycle.name' => plugin.ilm_policy, 'index.lifecycle.rollover_alias' => plugin.ilm_rollover_alias})
40
+ end
41
+
42
+ # Template name - if template_name set, use it
43
+ # if not and ILM is enabled, use the rollover alias
44
+ # else use the default value of template_name
45
+ def self.template_name(plugin)
46
+ plugin.ilm_in_use? && !plugin.original_params.key?('template_name') ? plugin.ilm_rollover_alias : plugin.template_name
47
+ end
48
+
49
+ def self.default_template_path(es_major_version)
50
+ template_version = es_major_version == 1 ? 2 : es_major_version
51
+ default_template_name = "elasticsearch-template-es#{template_version}x.json"
52
+ ::File.expand_path(default_template_name, ::File.dirname(__FILE__))
53
+ end
54
+
55
+ def self.read_template_file(template_path)
56
+ raise ArgumentError, "Template file '#{template_path}' could not be found!" unless ::File.exists?(template_path)
57
+ template_data = ::IO.read(template_path)
58
+ LogStash::Json.load(template_data)
59
+ end
60
+ end
61
+ end end end
@@ -0,0 +1,263 @@
1
+ # encoding: utf-8
2
+ require "logstash/namespace"
3
+ require "logstash/environment"
4
+ require "logstash/outputs/base"
5
+ require "logstash/json"
6
+ require "concurrent"
7
+ require "stud/buffer"
8
+ require "socket" # for Socket.gethostname
9
+ require "thread" # for safe queueing
10
+ require "uri" # for escaping user input
11
+ require "forwardable"
12
+
13
+ # .Compatibility Note
14
+ # [NOTE]
15
+ # ================================================================================
16
+ # Starting with Elasticsearch 5.3, there's an {ref}modules-http.html[HTTP setting]
17
+ # called `http.content_type.required`. If this option is set to `true`, and you
18
+ # are using Logstash 2.4 through 5.2, you need to update the Elasticsearch output
19
+ # plugin to version 6.2.5 or higher.
20
+ #
21
+ # ================================================================================
22
+ #
23
+ # This plugin is the recommended method of storing logs in Elasticsearch.
24
+ # If you plan on using the Kibana web interface, you'll want to use this output.
25
+ #
26
+ # This output only speaks the HTTP protocol. HTTP is the preferred protocol for interacting with Elasticsearch as of Logstash 2.0.
27
+ # We strongly encourage the use of HTTP over the node protocol for a number of reasons. HTTP is only marginally slower,
28
+ # yet far easier to administer and work with. When using the HTTP protocol one may upgrade Elasticsearch versions without having
29
+ # to upgrade Logstash in lock-step.
30
+ #
31
+ # You can learn more about Elasticsearch at <https://www.elastic.co/products/elasticsearch>
32
+ #
33
+ # ==== Template management for Elasticsearch 5.x
34
+ # Index template for this version (Logstash 5.0) has been changed to reflect Elasticsearch's mapping changes in version 5.0.
35
+ # Most importantly, the subfield for string multi-fields has changed from `.raw` to `.keyword` to match ES default
36
+ # behavior.
37
+ #
38
+ # ** Users installing ES 5.x and LS 5.x **
39
+ # This change will not affect you and you will continue to use the ES defaults.
40
+ #
41
+ # ** Users upgrading from LS 2.x to LS 5.x with ES 5.x **
42
+ # LS will not force upgrade the template, if `logstash` template already exists. This means you will still use
43
+ # `.raw` for sub-fields coming from 2.x. If you choose to use the new template, you will have to reindex your data after
44
+ # the new template is installed.
45
+ #
46
+ # ==== Retry Policy
47
+ #
48
+ # The retry policy has changed significantly in the 2.2.0 release.
49
+ # This plugin uses the Elasticsearch bulk API to optimize its imports into Elasticsearch. These requests may experience
50
+ # either partial or total failures.
51
+ #
52
+ # The following errors are retried infinitely:
53
+ #
54
+ # - Network errors (inability to connect)
55
+ # - 429 (Too many requests) and
56
+ # - 503 (Service unavailable) errors
57
+ #
58
+ # NOTE: 409 exceptions are no longer retried. Please set a higher `retry_on_conflict` value if you experience 409 exceptions.
59
+ # It is more performant for Elasticsearch to retry these exceptions than this plugin.
60
+ #
61
+ # ==== Batch Sizes ====
62
+ # This plugin attempts to send batches of events as a single request. However, if
63
+ # a request exceeds 20MB we will break it up until multiple batch requests. If a single document exceeds 20MB it will be sent as a single request.
64
+ #
65
+ # ==== DNS Caching
66
+ #
67
+ # This plugin uses the JVM to lookup DNS entries and is subject to the value of https://docs.oracle.com/javase/7/docs/technotes/guides/net/properties.html[networkaddress.cache.ttl],
68
+ # a global setting for the JVM.
69
+ #
70
+ # As an example, to set your DNS TTL to 1 second you would set
71
+ # the `LS_JAVA_OPTS` environment variable to `-Dnetworkaddress.cache.ttl=1`.
72
+ #
73
+ # Keep in mind that a connection with keepalive enabled will
74
+ # not reevaluate its DNS value while the keepalive is in effect.
75
+ #
76
+ # ==== HTTP Compression
77
+ #
78
+ # This plugin supports request and response compression. Response compression is enabled by default and
79
+ # for Elasticsearch versions 5.0 and later, the user doesn't have to set any configs in Elasticsearch for
80
+ # it to send back compressed response. For versions before 5.0, `http.compression` must be set to `true` in
81
+ # Elasticsearch[https://www.elastic.co/guide/en/elasticsearch/reference/current/modules-http.html#modules-http] to take advantage of response compression when using this plugin
82
+ #
83
+ # For requests compression, regardless of the Elasticsearch version, users have to enable `http_compression`
84
+ # setting in their Logstash config file.
85
+ #
86
+ class LogStash::Outputs::ElasticSearch < LogStash::Outputs::Base
87
+ declare_threadsafe!
88
+
89
+ require "logstash/outputs/elasticsearch/http_client"
90
+ require "logstash/outputs/elasticsearch/http_client_builder"
91
+ require "logstash/outputs/elasticsearch/common_configs"
92
+ require "logstash/outputs/elasticsearch/common"
93
+ require "logstash/outputs/elasticsearch/ilm"
94
+
95
+ # Protocol agnostic (i.e. non-http, non-java specific) configs go here
96
+ include(LogStash::Outputs::ElasticSearch::CommonConfigs)
97
+
98
+ # Protocol agnostic methods
99
+ include(LogStash::Outputs::ElasticSearch::Common)
100
+
101
+ # Methods for ILM support
102
+ include(LogStash::Outputs::ElasticSearch::Ilm)
103
+
104
+ config_name "elasticsearch"
105
+
106
+ # The Elasticsearch action to perform. Valid actions are:
107
+ #
108
+ # - index: indexes a document (an event from Logstash).
109
+ # - delete: deletes a document by id (An id is required for this action)
110
+ # - create: indexes a document, fails if a document by that id already exists in the index.
111
+ # - update: updates a document by id. Update has a special case where you can upsert -- update a
112
+ # document if not already present. See the `upsert` option. NOTE: This does not work and is not supported
113
+ # in Elasticsearch 1.x. Please upgrade to ES 2.x or greater to use this feature with Logstash!
114
+ # - A sprintf style string to change the action based on the content of the event. The value `%{[foo]}`
115
+ # would use the foo field for the action
116
+ #
117
+ # For more details on actions, check out the http://www.elastic.co/guide/en/elasticsearch/reference/current/docs-bulk.html[Elasticsearch bulk API documentation]
118
+ config :action, :validate => :string, :default => "index"
119
+
120
+ # Username to authenticate to a secure Elasticsearch cluster
121
+ config :user, :validate => :string
122
+ # Password to authenticate to a secure Elasticsearch cluster
123
+ config :password, :validate => :password
124
+
125
+ # Cloud authentication string ("<username>:<password>" format) is an alternative for the `user`/`password` configuration.
126
+ #
127
+ # For mode details, check out the https://www.elastic.co/guide/en/logstash/current/connecting-to-cloud.html#_cloud_auth[cloud documentation]
128
+ config :cloud_auth, :validate => :password
129
+
130
+ # HTTP Path at which the Elasticsearch server lives. Use this if you must run Elasticsearch behind a proxy that remaps
131
+ # the root path for the Elasticsearch HTTP API lives.
132
+ # Note that if you use paths as components of URLs in the 'hosts' field you may
133
+ # not also set this field. That will raise an error at startup
134
+ config :path, :validate => :string
135
+
136
+ # HTTP Path to perform the _bulk requests to
137
+ # this defaults to a concatenation of the path parameter and "_bulk"
138
+ config :bulk_path, :validate => :string
139
+
140
+ # Pass a set of key value pairs as the URL query string. This query string is added
141
+ # to every host listed in the 'hosts' configuration. If the 'hosts' list contains
142
+ # urls that already have query strings, the one specified here will be appended.
143
+ config :parameters, :validate => :hash
144
+
145
+ # Enable SSL/TLS secured communication to Elasticsearch cluster. Leaving this unspecified will use whatever scheme
146
+ # is specified in the URLs listed in 'hosts'. If no explicit protocol is specified plain HTTP will be used.
147
+ # If SSL is explicitly disabled here the plugin will refuse to start if an HTTPS URL is given in 'hosts'
148
+ config :ssl, :validate => :boolean
149
+
150
+ # Option to validate the server's certificate. Disabling this severely compromises security.
151
+ # For more information on disabling certificate verification please read
152
+ # https://www.cs.utexas.edu/~shmat/shmat_ccs12.pdf
153
+ config :ssl_certificate_verification, :validate => :boolean, :default => true
154
+
155
+ # The .cer or .pem file to validate the server's certificate
156
+ config :cacert, :validate => :path
157
+
158
+ # The JKS truststore to validate the server's certificate.
159
+ # Use either `:truststore` or `:cacert`
160
+ config :truststore, :validate => :path
161
+
162
+ # Set the truststore password
163
+ config :truststore_password, :validate => :password
164
+
165
+ # The keystore used to present a certificate to the server.
166
+ # It can be either .jks or .p12
167
+ config :keystore, :validate => :path
168
+
169
+ # Set the keystore password
170
+ config :keystore_password, :validate => :password
171
+
172
+ # This setting asks Elasticsearch for the list of all cluster nodes and adds them to the hosts list.
173
+ # Note: This will return ALL nodes with HTTP enabled (including master nodes!). If you use
174
+ # this with master nodes, you probably want to disable HTTP on them by setting
175
+ # `http.enabled` to false in their elasticsearch.yml. You can either use the `sniffing` option or
176
+ # manually enter multiple Elasticsearch hosts using the `hosts` parameter.
177
+ config :sniffing, :validate => :boolean, :default => false
178
+
179
+ # How long to wait, in seconds, between sniffing attempts
180
+ config :sniffing_delay, :validate => :number, :default => 5
181
+
182
+ # HTTP Path to be used for the sniffing requests
183
+ # the default value is computed by concatenating the path value and "_nodes/http"
184
+ # if sniffing_path is set it will be used as an absolute path
185
+ # do not use full URL here, only paths, e.g. "/sniff/_nodes/http"
186
+ config :sniffing_path, :validate => :string
187
+
188
+ # Set the address of a forward HTTP proxy.
189
+ # This used to accept hashes as arguments but now only accepts
190
+ # arguments of the URI type to prevent leaking credentials.
191
+ config :proxy, :validate => :uri
192
+
193
+ # Set the timeout, in seconds, for network operations and requests sent Elasticsearch. If
194
+ # a timeout occurs, the request will be retried.
195
+ config :timeout, :validate => :number, :default => 60
196
+
197
+ # Set the Elasticsearch errors in the whitelist that you don't want to log.
198
+ # A useful example is when you want to skip all 409 errors
199
+ # which are `document_already_exists_exception`.
200
+ config :failure_type_logging_whitelist, :validate => :array, :default => []
201
+
202
+ # While the output tries to reuse connections efficiently we have a maximum.
203
+ # This sets the maximum number of open connections the output will create.
204
+ # Setting this too low may mean frequently closing / opening connections
205
+ # which is bad.
206
+ config :pool_max, :validate => :number, :default => 1000
207
+
208
+ # While the output tries to reuse connections efficiently we have a maximum per endpoint.
209
+ # This sets the maximum number of open connections per endpoint the output will create.
210
+ # Setting this too low may mean frequently closing / opening connections
211
+ # which is bad.
212
+ config :pool_max_per_route, :validate => :number, :default => 100
213
+
214
+ # HTTP Path where a HEAD request is sent when a backend is marked down
215
+ # the request is sent in the background to see if it has come back again
216
+ # before it is once again eligible to service requests.
217
+ # If you have custom firewall rules you may need to change this
218
+ config :healthcheck_path, :validate => :string
219
+
220
+ # How frequently, in seconds, to wait between resurrection attempts.
221
+ # Resurrection is the process by which backend endpoints marked 'down' are checked
222
+ # to see if they have come back to life
223
+ config :resurrect_delay, :validate => :number, :default => 5
224
+
225
+ # How long to wait before checking if the connection is stale before executing a request on a connection using keepalive.
226
+ # You may want to set this lower, if you get connection errors regularly
227
+ # Quoting the Apache commons docs (this client is based Apache Commmons):
228
+ # 'Defines period of inactivity in milliseconds after which persistent connections must
229
+ # be re-validated prior to being leased to the consumer. Non-positive value passed to
230
+ # this method disables connection validation. This check helps detect connections that
231
+ # have become stale (half-closed) while kept inactive in the pool.'
232
+ # See https://hc.apache.org/httpcomponents-client-ga/httpclient/apidocs/org/apache/http/impl/conn/PoolingHttpClientConnectionManager.html#setValidateAfterInactivity(int)[these docs for more info]
233
+ config :validate_after_inactivity, :validate => :number, :default => 10000
234
+
235
+ # Enable gzip compression on requests. Note that response compression is on by default for Elasticsearch v5.0 and beyond
236
+ config :http_compression, :validate => :boolean, :default => false
237
+
238
+ # Custom Headers to send on each request to elasticsearch nodes
239
+ config :custom_headers, :validate => :hash, :default => {}
240
+
241
+ def build_client
242
+ params["metric"] = metric
243
+ @client ||= ::LogStash::Outputs::ElasticSearch::HttpClientBuilder.build(@logger, @hosts, params)
244
+ end
245
+
246
+ def close
247
+ @stopping.make_true
248
+ stop_template_installer
249
+ @client.close if @client
250
+ end
251
+
252
+ def self.oss?
253
+ LogStash::OSS
254
+ end
255
+
256
+ @@plugins = Gem::Specification.find_all{|spec| spec.name =~ /logstash-output-elasticsearch-/ }
257
+
258
+ @@plugins.each do |plugin|
259
+ name = plugin.name.split('-')[-1]
260
+ require "logstash/outputs/elasticsearch/#{name}"
261
+ end
262
+
263
+ end # class LogStash::Outputs::Elasticsearch
@@ -0,0 +1,33 @@
1
+ Gem::Specification.new do |s|
2
+ s.name = 'logstash-output-elasticsearch-test'
3
+ s.version = '10.3.0'
4
+ s.licenses = ['']
5
+ s.summary = "Stores logs in Elasticsearch"
6
+ s.description = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
7
+ s.authors = ["Elastic&Test"]
8
+ s.email = 'info@elastic.co'
9
+ s.homepage = "http://logstash.net/"
10
+ s.require_paths = ["lib"]
11
+
12
+ s.platform = RUBY_PLATFORM
13
+
14
+ # Files
15
+ s.files = Dir["lib/**/*","spec/**/*","*.gemspec","*.md","CONTRIBUTORS","Gemfile","LICENSE","NOTICE.TXT", "vendor/jar-dependencies/**/*.jar", "vendor/jar-dependencies/**/*.rb", "VERSION", "docs/**/*"]
16
+
17
+ # Tests
18
+ s.test_files = s.files.grep(%r{^(test|spec|features)/})
19
+
20
+ # Special flag to let us know this is actually a logstash plugin
21
+ s.metadata = { "logstash_plugin" => "true", "logstash_group" => "output" }
22
+
23
+ s.add_runtime_dependency "manticore", '>= 0.5.4', '< 1.0.0'
24
+ s.add_runtime_dependency 'stud', ['>= 0.0.17', '~> 0.0']
25
+ s.add_runtime_dependency 'cabin', ['~> 0.6']
26
+ s.add_runtime_dependency "logstash-core-plugin-api", ">= 1.60", "<= 2.99"
27
+
28
+ s.add_development_dependency 'logstash-codec-plain'
29
+ s.add_development_dependency 'logstash-devutils'
30
+ s.add_development_dependency 'flores'
31
+ # Still used in some specs, we should remove this ASAP
32
+ s.add_development_dependency 'elasticsearch'
33
+ end
@@ -0,0 +1,189 @@
1
+ require "logstash/devutils/rspec/spec_helper"
2
+ require 'manticore'
3
+ require 'elasticsearch'
4
+ require_relative "support/elasticsearch/api/actions/delete_ilm_policy"
5
+ require_relative "support/elasticsearch/api/actions/get_alias"
6
+ require_relative "support/elasticsearch/api/actions/put_alias"
7
+ require_relative "support/elasticsearch/api/actions/get_ilm_policy"
8
+ require_relative "support/elasticsearch/api/actions/put_ilm_policy"
9
+
10
+ require 'json'
11
+
12
+ unless defined?(LogStash::OSS)
13
+ LogStash::OSS = ENV['DISTRIBUTION'] != "default"
14
+ end
15
+
16
+ module ESHelper
17
+ def get_host_port
18
+ if ENV["INTEGRATION"] == "true"
19
+ "elasticsearch:9200"
20
+ else
21
+ "localhost:9200"
22
+ end
23
+ end
24
+
25
+ def get_client
26
+ Elasticsearch::Client.new(:hosts => [get_host_port])
27
+ end
28
+
29
+ def doc_type
30
+ if ESHelper.es_version_satisfies?(">=8")
31
+ nil
32
+ elsif ESHelper.es_version_satisfies?(">=7")
33
+ "_doc"
34
+ else
35
+ "doc"
36
+ end
37
+ end
38
+
39
+ def self.action_for_version(action)
40
+ action_params = action[1]
41
+ if ESHelper.es_version_satisfies?(">=8")
42
+ action_params.delete(:_type)
43
+ end
44
+ action[1] = action_params
45
+ action
46
+ end
47
+
48
+ def todays_date
49
+ Time.now.strftime("%Y.%m.%d")
50
+ end
51
+
52
+
53
+ def default_mapping_from_mappings(mappings)
54
+ if ESHelper.es_version_satisfies?(">=7")
55
+ mappings
56
+ else
57
+ mappings["_default_"]
58
+ end
59
+ end
60
+
61
+ def field_properties_from_template(template_name, field)
62
+ mappings = @es.indices.get_template(name: template_name)[template_name]["mappings"]
63
+ mapping = default_mapping_from_mappings(mappings)
64
+ mapping["properties"][field]["properties"]
65
+ end
66
+
67
+ def routing_field_name
68
+ if ESHelper.es_version_satisfies?(">=6")
69
+ :routing
70
+ else
71
+ :_routing
72
+ end
73
+ end
74
+
75
+ def self.es_version
76
+ RSpec.configuration.filter[:es_version] || ENV['ES_VERSION'] || ENV['ELASTIC_STACK_VERSION']
77
+ end
78
+
79
+ RSpec::Matchers.define :have_hits do |expected|
80
+ match do |actual|
81
+ if ESHelper.es_version_satisfies?(">=7")
82
+ expected == actual['hits']['total']['value']
83
+ else
84
+ expected == actual['hits']['total']
85
+ end
86
+ end
87
+ end
88
+
89
+ RSpec::Matchers.define :have_index_pattern do |expected|
90
+ match do |actual|
91
+ test_against = Array(actual['index_patterns'].nil? ? actual['template'] : actual['index_patterns'])
92
+ test_against.include?(expected)
93
+ end
94
+ end
95
+
96
+ def self.es_version_satisfies?(*requirement)
97
+ es_version = RSpec.configuration.filter[:es_version] || ENV['ES_VERSION'] || ENV['ELASTIC_STACK_VERSION']
98
+ if es_version.nil?
99
+ puts "Info: ES_VERSION, ELASTIC_STACK_VERSION or 'es_version' tag wasn't set. Returning false to all `es_version_satisfies?` call."
100
+ return false
101
+ end
102
+ es_release_version = Gem::Version.new(es_version).release
103
+ Gem::Requirement.new(requirement).satisfied_by?(es_release_version)
104
+ end
105
+
106
+ def clean(client)
107
+ client.indices.delete_template(:name => "*")
108
+ # This can fail if there are no indexes, ignore failure.
109
+ client.indices.delete(:index => "*") rescue nil
110
+ clean_ilm(client) if supports_ilm?(client)
111
+ end
112
+
113
+ def set_cluster_settings(client, cluster_settings)
114
+ client.cluster.put_settings(body: cluster_settings)
115
+ get_cluster_settings(client)
116
+ end
117
+
118
+ def get_cluster_settings(client)
119
+ client.cluster.get_settings
120
+ end
121
+
122
+ def get_policy(client, policy_name)
123
+ client.get_ilm_policy(name: policy_name)
124
+ end
125
+
126
+ def put_policy(client, policy_name, policy)
127
+ client.put_ilm_policy({:name => policy_name, :body=> policy})
128
+ end
129
+
130
+ def put_alias(client, the_alias, index)
131
+ body = {
132
+ "aliases" => {
133
+ index => {
134
+ "is_write_index"=> true
135
+ }
136
+ }
137
+ }
138
+ client.put_alias({name: the_alias, body: body})
139
+ end
140
+
141
+ def clean_ilm(client)
142
+ client.get_ilm_policy.each_key {|key| client.delete_ilm_policy(name: key)}
143
+ end
144
+
145
+ def supports_ilm?(client)
146
+ begin
147
+ client.get_ilm_policy
148
+ true
149
+ rescue
150
+ false
151
+ end
152
+ end
153
+
154
+ def max_docs_policy(max_docs)
155
+ {
156
+ "policy" => {
157
+ "phases"=> {
158
+ "hot" => {
159
+ "actions" => {
160
+ "rollover" => {
161
+ "max_docs" => max_docs
162
+ }
163
+ }
164
+ }
165
+ }
166
+ }
167
+ }
168
+ end
169
+
170
+ def max_age_policy(max_age)
171
+ {
172
+ "policy" => {
173
+ "phases"=> {
174
+ "hot" => {
175
+ "actions" => {
176
+ "rollover" => {
177
+ "max_age" => max_age
178
+ }
179
+ }
180
+ }
181
+ }
182
+ }
183
+ }
184
+ end
185
+ end
186
+
187
+ RSpec.configure do |config|
188
+ config.include ESHelper
189
+ end
@@ -0,0 +1,27 @@
1
+ {
2
+ "cluster_name" : "dev",
3
+ "nodes" : {
4
+ "Ur_68iBvTlm7Xr1HgSsh6w" : {
5
+ "name" : "dev-es-master01",
6
+ "transport_address" : "http://localhost:9200",
7
+ "host" : "127.0.0.1",
8
+ "ip" : "127.0.0.1",
9
+ "version" : "2.4.6",
10
+ "build" : "5376dca"
11
+ },
12
+ "sari4do3RG-mgh2CIZeHwA" : {
13
+ "name" : "dev-es-data01",
14
+ "transport_address" : "http://localhost:9201",
15
+ "host" : "127.0.0.1",
16
+ "ip" : "127.0.0.1",
17
+ "version" : "2.4.6",
18
+ "build" : "5376dca",
19
+ "http_address" : "127.0.0.1:9201",
20
+ "http" : {
21
+ "bound_address" : [ "[::1]:9201", "127.0.0.1:9201" ],
22
+ "publish_address" : "127.0.0.1:9201",
23
+ "max_content_length_in_bytes" : 104857600
24
+ }
25
+ }
26
+ }
27
+ }
@@ -0,0 +1,81 @@
1
+ {
2
+ "_nodes" : {
3
+ "total" : 3,
4
+ "successful" : 3,
5
+ "failed" : 0
6
+ },
7
+ "cluster_name" : "dev",
8
+ "nodes" : {
9
+ "Ur_68iBvTlm7Xr1HgSsh6w" : {
10
+ "name" : "dev-es-master01",
11
+ "transport_address" : "http://localhost:9200",
12
+ "host" : "localhost",
13
+ "ip" : "127.0.0.1",
14
+ "version" : "5.5.1",
15
+ "build_hash" : "19c13d0",
16
+ "roles" : [
17
+ "master"
18
+ ],
19
+ "http" : {
20
+ "bound_address" : [
21
+ "[::]:9200"
22
+ ],
23
+ "publish_address" : "127.0.0.1:9200",
24
+ "max_content_length_in_bytes" : 104857600
25
+ }
26
+ },
27
+ "sari4do3RG-mgh2CIZeHwA" : {
28
+ "name" : "dev-es-data01",
29
+ "transport_address" : "http://localhost:9201",
30
+ "host" : "localhost",
31
+ "ip" : "127.0.0.1",
32
+ "version" : "5.5.1",
33
+ "build_hash" : "19c13d0",
34
+ "roles" : [
35
+ "data"
36
+ ],
37
+ "http" : {
38
+ "bound_address" : [
39
+ "[::]:9200"
40
+ ],
41
+ "publish_address" : "127.0.0.1:9201",
42
+ "max_content_length_in_bytes" : 104857600
43
+ }
44
+ },
45
+ "Rjy1WL66RHm4fyzXA8PCGQ" : {
46
+ "name" : "dev-es-datamaster01",
47
+ "transport_address" : "http://localhost:9202",
48
+ "host" : "localhost",
49
+ "ip" : "127.0.0.1",
50
+ "version" : "5.5.1",
51
+ "build_hash" : "19c13d0",
52
+ "roles" : [
53
+ "data",
54
+ "master"
55
+ ],
56
+ "http" : {
57
+ "bound_address" : [
58
+ "[::]:9200"
59
+ ],
60
+ "publish_address" : "127.0.0.1:9202",
61
+ "max_content_length_in_bytes" : 104857600
62
+ }
63
+ },
64
+ "OguP_obcT_S9JYNB8SKKgQ" : {
65
+ "name" : "dev-es-coordinator01",
66
+ "transport_address" : "http://localhost:9203",
67
+ "host" : "localhost",
68
+ "ip" : "127.0.0.1",
69
+ "version" : "5.5.1",
70
+ "build_hash" : "19c13d0",
71
+ "roles" : [ ],
72
+ "http" : {
73
+ "bound_address" : [
74
+ "[::]:9200"
75
+ ],
76
+ "publish_address" : "127.0.0.1:9203",
77
+ "max_content_length_in_bytes" : 104857600
78
+ }
79
+ }
80
+ }
81
+ }