logstash-output-elasticsearch 10.8.6-java → 11.0.3-java

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (49) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +17 -0
  3. data/docs/index.asciidoc +132 -22
  4. data/lib/logstash/outputs/elasticsearch.rb +125 -64
  5. data/lib/logstash/outputs/elasticsearch/data_stream_support.rb +233 -0
  6. data/lib/logstash/outputs/elasticsearch/http_client.rb +9 -7
  7. data/lib/logstash/outputs/elasticsearch/http_client/pool.rb +49 -62
  8. data/lib/logstash/outputs/elasticsearch/ilm.rb +13 -45
  9. data/lib/logstash/outputs/elasticsearch/license_checker.rb +26 -23
  10. data/lib/logstash/outputs/elasticsearch/template_manager.rb +4 -6
  11. data/lib/logstash/outputs/elasticsearch/templates/ecs-v1/elasticsearch-8x.json +1 -0
  12. data/lib/logstash/plugin_mixins/elasticsearch/api_configs.rb +157 -153
  13. data/lib/logstash/plugin_mixins/elasticsearch/common.rb +71 -58
  14. data/logstash-output-elasticsearch.gemspec +3 -3
  15. data/spec/es_spec_helper.rb +7 -12
  16. data/spec/fixtures/_nodes/{5x_6x.json → 6x.json} +5 -5
  17. data/spec/integration/outputs/compressed_indexing_spec.rb +47 -46
  18. data/spec/integration/outputs/data_stream_spec.rb +61 -0
  19. data/spec/integration/outputs/delete_spec.rb +49 -51
  20. data/spec/integration/outputs/ilm_spec.rb +236 -248
  21. data/spec/integration/outputs/index_spec.rb +5 -2
  22. data/spec/integration/outputs/index_version_spec.rb +78 -82
  23. data/spec/integration/outputs/ingest_pipeline_spec.rb +58 -58
  24. data/spec/integration/outputs/painless_update_spec.rb +74 -164
  25. data/spec/integration/outputs/parent_spec.rb +67 -75
  26. data/spec/integration/outputs/retry_spec.rb +6 -6
  27. data/spec/integration/outputs/sniffer_spec.rb +15 -54
  28. data/spec/integration/outputs/templates_spec.rb +79 -81
  29. data/spec/integration/outputs/update_spec.rb +99 -101
  30. data/spec/spec_helper.rb +10 -0
  31. data/spec/unit/outputs/elasticsearch/data_stream_support_spec.rb +528 -0
  32. data/spec/unit/outputs/elasticsearch/http_client/manticore_adapter_spec.rb +1 -0
  33. data/spec/unit/outputs/elasticsearch/http_client/pool_spec.rb +36 -29
  34. data/spec/unit/outputs/elasticsearch/http_client_spec.rb +2 -3
  35. data/spec/unit/outputs/elasticsearch/template_manager_spec.rb +10 -12
  36. data/spec/unit/outputs/elasticsearch_proxy_spec.rb +1 -2
  37. data/spec/unit/outputs/elasticsearch_spec.rb +176 -41
  38. data/spec/unit/outputs/elasticsearch_ssl_spec.rb +1 -2
  39. data/spec/unit/outputs/error_whitelist_spec.rb +3 -2
  40. data/spec/unit/outputs/license_check_spec.rb +0 -16
  41. metadata +29 -36
  42. data/lib/logstash/outputs/elasticsearch/templates/ecs-disabled/elasticsearch-2x.json +0 -95
  43. data/lib/logstash/outputs/elasticsearch/templates/ecs-disabled/elasticsearch-5x.json +0 -46
  44. data/spec/fixtures/_nodes/2x_1x.json +0 -27
  45. data/spec/fixtures/scripts/groovy/scripted_update.groovy +0 -2
  46. data/spec/fixtures/scripts/groovy/scripted_update_nested.groovy +0 -2
  47. data/spec/fixtures/scripts/groovy/scripted_upsert.groovy +0 -2
  48. data/spec/integration/outputs/groovy_update_spec.rb +0 -150
  49. data/spec/integration/outputs/templates_5x_spec.rb +0 -98
@@ -1,98 +0,0 @@
1
- require_relative "../../../spec/es_spec_helper"
2
-
3
- if ESHelper.es_version_satisfies?(">= 5")
4
- describe "index template expected behavior for 5.x", :integration => true do
5
- subject! do
6
- require "logstash/outputs/elasticsearch"
7
- settings = {
8
- "manage_template" => true,
9
- "template_overwrite" => true,
10
- "hosts" => "#{get_host_port()}"
11
- }
12
- next LogStash::Outputs::ElasticSearch.new(settings)
13
- end
14
-
15
- before :each do
16
- # Delete all templates first.
17
- require "elasticsearch"
18
-
19
- # Clean ES of data before we start.
20
- @es = get_client
21
- @es.indices.delete_template(:name => "*")
22
-
23
- # This can fail if there are no indexes, ignore failure.
24
- @es.indices.delete(:index => "*") rescue nil
25
-
26
- subject.register
27
-
28
- subject.multi_receive([
29
- LogStash::Event.new("message" => "sample message here"),
30
- LogStash::Event.new("somemessage" => { "message" => "sample nested message here" }),
31
- LogStash::Event.new("somevalue" => 100),
32
- LogStash::Event.new("somevalue" => 10),
33
- LogStash::Event.new("somevalue" => 1),
34
- LogStash::Event.new("country" => "us"),
35
- LogStash::Event.new("country" => "at"),
36
- LogStash::Event.new("geoip" => { "location" => [ 0.0, 0.0 ] })
37
- ])
38
-
39
- @es.indices.refresh
40
-
41
- # Wait or fail until everything's indexed.
42
- Stud::try(20.times) do
43
- r = @es.search(index: 'logstash-*')
44
- expect(r).to have_hits(8)
45
- end
46
- end
47
-
48
- it "permits phrase searching on string fields" do
49
- results = @es.search(:q => "message:\"sample message\"")
50
- expect(results).to have_hits(1)
51
- expect(results["hits"]["hits"][0]["_source"]["message"]).to eq("sample message here")
52
- end
53
-
54
- it "numbers dynamically map to a numeric type and permit range queries" do
55
- results = @es.search(:q => "somevalue:[5 TO 105]")
56
- expect(results).to have_hits(2)
57
-
58
- values = results["hits"]["hits"].collect { |r| r["_source"]["somevalue"] }
59
- expect(values).to include(10)
60
- expect(values).to include(100)
61
- expect(values).to_not include(1)
62
- end
63
-
64
- it "does not create .keyword field for top-level message field" do
65
- results = @es.search(:q => "message.keyword:\"sample message here\"")
66
- expect(results).to have_hits(0)
67
- end
68
-
69
- it "creates .keyword field for nested message fields" do
70
- results = @es.search(:q => "somemessage.message.keyword:\"sample nested message here\"")
71
- expect(results).to have_hits(1)
72
- end
73
-
74
- it "creates .keyword field from any string field which is not_analyzed" do
75
- results = @es.search(:q => "country.keyword:\"us\"")
76
- expect(results).to have_hits(1)
77
- expect(results["hits"]["hits"][0]["_source"]["country"]).to eq("us")
78
-
79
- # partial or terms should not work.
80
- results = @es.search(:q => "country.keyword:\"u\"")
81
- expect(results).to have_hits(0)
82
- end
83
-
84
- it "make [geoip][location] a geo_point" do
85
- expect(field_properties_from_template("logstash", "geoip")["location"]["type"]).to eq("geo_point")
86
- end
87
-
88
- it "aggregate .keyword results correctly " do
89
- results = @es.search(:body => { "aggregations" => { "my_agg" => { "terms" => { "field" => "country.keyword" } } } })["aggregations"]["my_agg"]
90
- terms = results["buckets"].collect { |b| b["key"] }
91
-
92
- expect(terms).to include("us")
93
-
94
- # 'at' is a stopword, make sure stopwords are not ignored.
95
- expect(terms).to include("at")
96
- end
97
- end
98
- end