logstash-output-opensearch 1.1.0-java → 2.0.0-java

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (55) hide show
  1. checksums.yaml +4 -4
  2. checksums.yaml.gz.sig +0 -0
  3. data/COMPATIBILITY.md +5 -3
  4. data/CONTRIBUTING.md +26 -0
  5. data/DEVELOPER_GUIDE.md +9 -0
  6. data/Gemfile +1 -1
  7. data/MAINTAINERS.md +3 -0
  8. data/README.md +90 -3
  9. data/docs/ecs_compatibility.md +42 -0
  10. data/lib/logstash/outputs/opensearch/http_client/manticore_adapter.rb +122 -10
  11. data/lib/logstash/outputs/opensearch/http_client/pool.rb +11 -2
  12. data/lib/logstash/outputs/opensearch/http_client.rb +22 -25
  13. data/lib/logstash/outputs/opensearch/http_client_builder.rb +9 -3
  14. data/lib/logstash/outputs/opensearch/template_manager.rb +6 -5
  15. data/lib/logstash/outputs/opensearch/templates/ecs-disabled/1x_index.json +66 -0
  16. data/lib/logstash/outputs/opensearch/templates/ecs-disabled/2x.json +44 -0
  17. data/lib/logstash/outputs/opensearch/templates/ecs-disabled/2x_index.json +66 -0
  18. data/lib/logstash/outputs/opensearch/templates/ecs-disabled/7x_index.json +66 -0
  19. data/lib/logstash/outputs/opensearch/templates/ecs-v1/1x.json +3629 -0
  20. data/lib/logstash/outputs/opensearch/templates/ecs-v1/1x_index.json +3631 -0
  21. data/lib/logstash/outputs/opensearch/templates/ecs-v1/2x.json +3629 -0
  22. data/lib/logstash/outputs/opensearch/templates/ecs-v1/2x_index.json +3631 -0
  23. data/lib/logstash/outputs/opensearch/templates/ecs-v1/7x.json +3629 -0
  24. data/lib/logstash/outputs/opensearch/templates/ecs-v1/7x_index.json +3631 -0
  25. data/lib/logstash/outputs/opensearch/templates/ecs-v8/1x.json +5252 -0
  26. data/lib/logstash/outputs/opensearch/templates/ecs-v8/1x_index.json +5254 -0
  27. data/lib/logstash/outputs/opensearch/templates/ecs-v8/2x.json +5252 -0
  28. data/lib/logstash/outputs/opensearch/templates/ecs-v8/2x_index.json +5254 -0
  29. data/lib/logstash/outputs/opensearch/templates/ecs-v8/7x.json +5252 -0
  30. data/lib/logstash/outputs/opensearch/templates/ecs-v8/7x_index.json +5254 -0
  31. data/lib/logstash/outputs/opensearch.rb +10 -3
  32. data/lib/logstash/plugin_mixins/opensearch/api_configs.rb +26 -0
  33. data/logstash-output-opensearch.gemspec +19 -3
  34. data/spec/integration/outputs/compressed_indexing_spec.rb +10 -4
  35. data/spec/integration/outputs/create_spec.rb +7 -7
  36. data/spec/integration/outputs/delete_spec.rb +8 -8
  37. data/spec/integration/outputs/index_spec.rb +52 -10
  38. data/spec/integration/outputs/index_version_spec.rb +11 -11
  39. data/spec/integration/outputs/ingest_pipeline_spec.rb +2 -3
  40. data/spec/integration/outputs/metrics_spec.rb +0 -2
  41. data/spec/integration/outputs/no_opensearch_on_startup_spec.rb +0 -1
  42. data/spec/integration/outputs/painless_update_spec.rb +10 -10
  43. data/spec/integration/outputs/parent_spec.rb +1 -1
  44. data/spec/integration/outputs/retry_spec.rb +2 -2
  45. data/spec/integration/outputs/sniffer_spec.rb +2 -2
  46. data/spec/integration/outputs/templates_spec.rb +83 -59
  47. data/spec/integration/outputs/update_spec.rb +14 -14
  48. data/spec/opensearch_spec_helper.rb +12 -2
  49. data/spec/unit/outputs/opensearch/http_client/manticore_adapter_spec.rb +86 -3
  50. data/spec/unit/outputs/opensearch/http_client_spec.rb +26 -5
  51. data/spec/unit/outputs/opensearch/template_manager_spec.rb +11 -4
  52. data/spec/unit/outputs/opensearch_spec.rb +18 -2
  53. data.tar.gz.sig +0 -0
  54. metadata +74 -9
  55. metadata.gz.sig +0 -0
data/lib/logstash/outputs/opensearch/template_manager.rb CHANGED
@@ -18,7 +18,7 @@ module LogStash; module Outputs; class OpenSearch
18
18
  else
19
19
  plugin.logger.info("Using a default mapping template", :version => plugin.maximum_seen_major_version,
20
20
  :ecs_compatibility => plugin.ecs_compatibility)
21
- template = load_default_template(plugin.maximum_seen_major_version, plugin.ecs_compatibility)
21
+ template = load_default_template(plugin.maximum_seen_major_version, plugin.ecs_compatibility, plugin.client.legacy_template?)
22
22
  end
23
23
 
24
24
  plugin.logger.debug("Attempting to install template", template: template)
@@ -26,8 +26,8 @@ module LogStash; module Outputs; class OpenSearch
26
26
  end
27
27
 
28
28
  private
29
- def self.load_default_template(major_version, ecs_compatibility)
30
- template_path = default_template_path(major_version, ecs_compatibility)
29
+ def self.load_default_template(major_version, ecs_compatibility, legacy_template)
30
+ template_path = default_template_path(major_version, ecs_compatibility, legacy_template)
31
31
  read_template_file(template_path)
32
32
  rescue => e
33
33
  fail "Failed to load default template for OpenSearch v#{major_version} with ECS #{ecs_compatibility}; caused by: #{e.inspect}"
@@ -45,9 +45,10 @@ module LogStash; module Outputs; class OpenSearch
45
45
  plugin.template_name
46
46
  end
47
47
 
48
- def self.default_template_path(major_version, ecs_compatibility=:disabled)
48
+ def self.default_template_path(major_version, ecs_compatibility=:disabled, legacy_template=true)
49
49
  template_version = major_version
50
- default_template_name = "templates/ecs-#{ecs_compatibility}/#{template_version}x.json"
50
+ suffix = legacy_template ? "" : "_index"
51
+ default_template_name = "templates/ecs-#{ecs_compatibility}/#{template_version}x#{suffix}.json"
51
52
  ::File.expand_path(default_template_name, ::File.dirname(__FILE__))
52
53
  end
53
54
 
data/lib/logstash/outputs/opensearch/templates/ecs-disabled/1x_index.json ADDED
@@ -0,0 +1,66 @@
1
+ {
2
+ "index_patterns": "logstash-*",
3
+ "version": 60001,
4
+ "priority": 10,
5
+ "template": {
6
+ "settings": {
7
+ "index.refresh_interval": "5s",
8
+ "number_of_shards": 1
9
+ },
10
+ "mappings": {
11
+ "dynamic_templates": [
12
+ {
13
+ "message_field": {
14
+ "path_match": "message",
15
+ "match_mapping_type": "string",
16
+ "mapping": {
17
+ "type": "text",
18
+ "norms": false
19
+ }
20
+ }
21
+ },
22
+ {
23
+ "string_fields": {
24
+ "match": "*",
25
+ "match_mapping_type": "string",
26
+ "mapping": {
27
+ "type": "text",
28
+ "norms": false,
29
+ "fields": {
30
+ "keyword": {
31
+ "type": "keyword",
32
+ "ignore_above": 256
33
+ }
34
+ }
35
+ }
36
+ }
37
+ }
38
+ ],
39
+ "properties": {
40
+ "@timestamp": {
41
+ "type": "date"
42
+ },
43
+ "@version": {
44
+ "type": "keyword"
45
+ },
46
+ "geoip": {
47
+ "dynamic": true,
48
+ "properties": {
49
+ "ip": {
50
+ "type": "ip"
51
+ },
52
+ "location": {
53
+ "type": "geo_point"
54
+ },
55
+ "latitude": {
56
+ "type": "half_float"
57
+ },
58
+ "longitude": {
59
+ "type": "half_float"
60
+ }
61
+ }
62
+ }
63
+ }
64
+ }
65
+ }
66
+ }
data/lib/logstash/outputs/opensearch/templates/ecs-disabled/2x.json ADDED
@@ -0,0 +1,44 @@
1
+ {
2
+ "index_patterns" : "logstash-*",
3
+ "version" : 60001,
4
+ "settings" : {
5
+ "index.refresh_interval" : "5s",
6
+ "number_of_shards": 1
7
+ },
8
+ "mappings" : {
9
+ "dynamic_templates" : [ {
10
+ "message_field" : {
11
+ "path_match" : "message",
12
+ "match_mapping_type" : "string",
13
+ "mapping" : {
14
+ "type" : "text",
15
+ "norms" : false
16
+ }
17
+ }
18
+ }, {
19
+ "string_fields" : {
20
+ "match" : "*",
21
+ "match_mapping_type" : "string",
22
+ "mapping" : {
23
+ "type" : "text", "norms" : false,
24
+ "fields" : {
25
+ "keyword" : { "type": "keyword", "ignore_above": 256 }
26
+ }
27
+ }
28
+ }
29
+ } ],
30
+ "properties" : {
31
+ "@timestamp": { "type": "date"},
32
+ "@version": { "type": "keyword"},
33
+ "geoip" : {
34
+ "dynamic": true,
35
+ "properties" : {
36
+ "ip": { "type": "ip" },
37
+ "location" : { "type" : "geo_point" },
38
+ "latitude" : { "type" : "half_float" },
39
+ "longitude" : { "type" : "half_float" }
40
+ }
41
+ }
42
+ }
43
+ }
44
+ }
data/lib/logstash/outputs/opensearch/templates/ecs-disabled/2x_index.json ADDED
@@ -0,0 +1,66 @@
1
+ {
2
+ "index_patterns": "logstash-*",
3
+ "version": 60001,
4
+ "priority": 10,
5
+ "template": {
6
+ "settings": {
7
+ "index.refresh_interval": "5s",
8
+ "number_of_shards": 1
9
+ },
10
+ "mappings": {
11
+ "dynamic_templates": [
12
+ {
13
+ "message_field": {
14
+ "path_match": "message",
15
+ "match_mapping_type": "string",
16
+ "mapping": {
17
+ "type": "text",
18
+ "norms": false
19
+ }
20
+ }
21
+ },
22
+ {
23
+ "string_fields": {
24
+ "match": "*",
25
+ "match_mapping_type": "string",
26
+ "mapping": {
27
+ "type": "text",
28
+ "norms": false,
29
+ "fields": {
30
+ "keyword": {
31
+ "type": "keyword",
32
+ "ignore_above": 256
33
+ }
34
+ }
35
+ }
36
+ }
37
+ }
38
+ ],
39
+ "properties": {
40
+ "@timestamp": {
41
+ "type": "date"
42
+ },
43
+ "@version": {
44
+ "type": "keyword"
45
+ },
46
+ "geoip": {
47
+ "dynamic": true,
48
+ "properties": {
49
+ "ip": {
50
+ "type": "ip"
51
+ },
52
+ "location": {
53
+ "type": "geo_point"
54
+ },
55
+ "latitude": {
56
+ "type": "half_float"
57
+ },
58
+ "longitude": {
59
+ "type": "half_float"
60
+ }
61
+ }
62
+ }
63
+ }
64
+ }
65
+ }
66
+ }
data/lib/logstash/outputs/opensearch/templates/ecs-disabled/7x_index.json ADDED
@@ -0,0 +1,66 @@
1
+ {
2
+ "index_patterns": "logstash-*",
3
+ "version": 60001,
4
+ "priority": 10,
5
+ "template": {
6
+ "settings": {
7
+ "index.refresh_interval": "5s",
8
+ "number_of_shards": 1
9
+ },
10
+ "mappings": {
11
+ "dynamic_templates": [
12
+ {
13
+ "message_field": {
14
+ "path_match": "message",
15
+ "match_mapping_type": "string",
16
+ "mapping": {
17
+ "type": "text",
18
+ "norms": false
19
+ }
20
+ }
21
+ },
22
+ {
23
+ "string_fields": {
24
+ "match": "*",
25
+ "match_mapping_type": "string",
26
+ "mapping": {
27
+ "type": "text",
28
+ "norms": false,
29
+ "fields": {
30
+ "keyword": {
31
+ "type": "keyword",
32
+ "ignore_above": 256
33
+ }
34
+ }
35
+ }
36
+ }
37
+ }
38
+ ],
39
+ "properties": {
40
+ "@timestamp": {
41
+ "type": "date"
42
+ },
43
+ "@version": {
44
+ "type": "keyword"
45
+ },
46
+ "geoip": {
47
+ "dynamic": true,
48
+ "properties": {
49
+ "ip": {
50
+ "type": "ip"
51
+ },
52
+ "location": {
53
+ "type": "geo_point"
54
+ },
55
+ "latitude": {
56
+ "type": "half_float"
57
+ },
58
+ "longitude": {
59
+ "type": "half_float"
60
+ }
61
+ }
62
+ }
63
+ }
64
+ }
65
+ }
66
+ }