logstash-output-elasticsearch 11.0.2-java → 11.0.3-java

data/lib/logstash/outputs/elasticsearch/templates/ecs-disabled/elasticsearch-2x.json

@@ -1,95 +0,0 @@
-{
-  "template" : "logstash-*",
-  "settings" : {
-    "index.refresh_interval" : "5s"
-  },
-  "mappings" : {
-    "_default_" : {
-      "_all" : {"enabled" : true, "omit_norms" : true},
-      "dynamic_templates" : [ {
-        "message_field" : {
-          "path_match" : "message",
-          "match_mapping_type" : "string",
-          "mapping" : {
-            "type" : "string", "index" : "analyzed", "omit_norms" : true,
-            "fielddata" : { "format" : "disabled" }
-          }
-        }
-      }, {
-        "string_fields" : {
-          "match" : "*",
-          "match_mapping_type" : "string",
-          "mapping" : {
-            "type" : "string", "index" : "analyzed", "omit_norms" : true,
-            "fielddata" : { "format" : "disabled" },
-            "fields" : {
-              "raw" : {"type": "string", "index" : "not_analyzed", "doc_values" : true, "ignore_above" : 256}
-            }
-          }
-        }
-      }, {
-        "float_fields" : {
-          "match" : "*",
-          "match_mapping_type" : "float",
-          "mapping" : { "type" : "float", "doc_values" : true }
-        }
-      }, {
-        "double_fields" : {
-          "match" : "*",
-          "match_mapping_type" : "double",
-          "mapping" : { "type" : "double", "doc_values" : true }
-        }
-      }, {
-        "byte_fields" : {
-          "match" : "*",
-          "match_mapping_type" : "byte",
-          "mapping" : { "type" : "byte", "doc_values" : true }
-        }
-      }, {
-        "short_fields" : {
-          "match" : "*",
-          "match_mapping_type" : "short",
-          "mapping" : { "type" : "short", "doc_values" : true }
-        }
-      }, {
-        "integer_fields" : {
-          "match" : "*",
-          "match_mapping_type" : "integer",
-          "mapping" : { "type" : "integer", "doc_values" : true }
-        }
-      }, {
-        "long_fields" : {
-          "match" : "*",
-          "match_mapping_type" : "long",
-          "mapping" : { "type" : "long", "doc_values" : true }
-        }
-      }, {
-        "date_fields" : {
-          "match" : "*",
-          "match_mapping_type" : "date",
-          "mapping" : { "type" : "date", "doc_values" : true }
-        }
-      }, {
-        "geo_point_fields" : {
-          "match" : "*",
-          "match_mapping_type" : "geo_point",
-          "mapping" : { "type" : "geo_point", "doc_values" : true }
-        }
-      } ],
-      "properties" : {
-        "@timestamp": { "type": "date", "doc_values" : true },
-        "@version": { "type": "string", "index": "not_analyzed", "doc_values" : true },
-        "geoip"  : {
-          "type" : "object",
-          "dynamic": true,
-          "properties" : {
-            "ip": { "type": "ip", "doc_values" : true },
-            "location" : { "type" : "geo_point", "doc_values" : true },
-            "latitude" : { "type" : "float", "doc_values" : true },
-            "longitude" : { "type" : "float", "doc_values" : true }
-          }
-        }
-      }
-    }
-  }
-}

data/lib/logstash/outputs/elasticsearch/templates/ecs-disabled/elasticsearch-5x.json

@@ -1,46 +0,0 @@
-{
-  "template" : "logstash-*",
-  "version" : 50001,
-  "settings" : {
-    "index.refresh_interval" : "5s"
-  },
-  "mappings" : {
-    "_default_" : {
-      "_all" : {"enabled" : true, "norms" : false},
-      "dynamic_templates" : [ {
-        "message_field" : {
-          "path_match" : "message",
-          "match_mapping_type" : "string",
-          "mapping" : {
-            "type" : "text",
-            "norms" : false
-          }
-        }
-      }, {
-        "string_fields" : {
-          "match" : "*",
-          "match_mapping_type" : "string",
-          "mapping" : {
-            "type" : "text", "norms" : false,
-            "fields" : {
-              "keyword" : { "type": "keyword", "ignore_above": 256 }
-            }
-          }
-        }
-      } ],
-      "properties" : {
-        "@timestamp": { "type": "date", "include_in_all": false },
-        "@version": { "type": "keyword", "include_in_all": false },
-        "geoip"  : {
-          "dynamic": true,
-          "properties" : {
-            "ip": { "type": "ip" },
-            "location" : { "type" : "geo_point" },
-            "latitude" : { "type" : "half_float" },
-            "longitude" : { "type" : "half_float" }
-          }
-        }
-      }
-    }
-  }
-}

data/spec/fixtures/_nodes/2x_1x.json

@@ -1,27 +0,0 @@
-{
-  "cluster_name" : "dev",
-  "nodes" : {
-    "Ur_68iBvTlm7Xr1HgSsh6w" : {
-      "name" : "dev-es-master01",
-      "transport_address" : "http://localhost:9200",
-      "host" : "127.0.0.1",
-      "ip" : "127.0.0.1",
-      "version" : "2.4.6",
-      "build" : "5376dca"
-    },
-    "sari4do3RG-mgh2CIZeHwA" : {
-      "name" : "dev-es-data01",
-      "transport_address" : "http://localhost:9201",
-      "host" : "127.0.0.1",
-      "ip" : "127.0.0.1",
-      "version" : "2.4.6",
-      "build" : "5376dca",
-      "http_address" : "127.0.0.1:9201",
-      "http" : {
-        "bound_address" : [ "[::1]:9201", "127.0.0.1:9201" ],
-        "publish_address" : "127.0.0.1:9201",
-        "max_content_length_in_bytes" : 104857600
-      }
-    }
-  }
-}

data/spec/fixtures/scripts/groovy/scripted_update.groovy

@@ -1,2 +0,0 @@
-ctx._source.counter += event["count"]
-

data/spec/fixtures/scripts/groovy/scripted_update_nested.groovy

@@ -1,2 +0,0 @@
-ctx._source.counter += event["data"]["count"]
-

data/spec/fixtures/scripts/groovy/scripted_upsert.groovy

@@ -1,2 +0,0 @@
-ctx._source.counter = event["counter"]
-

data/spec/integration/outputs/groovy_update_spec.rb

@@ -1,150 +0,0 @@
-require_relative "../../../spec/es_spec_helper"
-
-if ESHelper.es_version_satisfies?('>= 2', '< 6')
-  describe "Update actions using groovy scripts", :integration => true, :update_tests => 'groovy' do
-    require "logstash/outputs/elasticsearch"
-
-    def get_es_output( options={} )
-      settings = {
-        "manage_template" => true,
-        "index" => "logstash-update",
-        "template_overwrite" => true,
-        "hosts" => get_host_port(),
-        "action" => "update",
-        "script_lang" => "groovy"
-      }
-      LogStash::Outputs::ElasticSearch.new(settings.merge!(options))
-    end
-
-    before :each do
-      @es = get_client
-      # Delete all templates first.
-      # Clean ES of data before we start.
-      @es.indices.delete_template(:name => "*")
-      # This can fail if there are no indexes, ignore failure.
-      @es.indices.delete(:index => "*") rescue nil
-      @es.index(
-        :index => 'logstash-update',
-        :type => doc_type,
-        :id => "123",
-        :body => { :message => 'Test', :counter => 1 }
-      )
-      @es.indices.refresh
-    end
-
-    context "scripted updates" do
-      it "should increment a counter with event/doc 'count' variable" do
-        subject = get_es_output({ 'document_id' => "123", 'script' => 'scripted_update', 'script_type' => 'file' })
-        subject.register
-        subject.multi_receive([LogStash::Event.new("count" => 2)])
-        r = @es.get(:index => 'logstash-update', :type => doc_type, :id => "123", :refresh => true)
-        expect(r["_source"]["counter"]).to eq(3)
-      end
-
-      it "should increment a counter with event/doc '[data][count]' nested variable" do
-        subject = get_es_output({ 'document_id' => "123", 'script' => 'scripted_update_nested', 'script_type' => 'file' })
-        subject.register
-        subject.multi_receive([LogStash::Event.new("data" => { "count" => 3 })])
-        r = @es.get(:index => 'logstash-update', :type => doc_type, :id => "123", :refresh => true)
-        expect(r["_source"]["counter"]).to eq(4)
-      end
-
-      it "should increment a counter with event/doc 'count' variable with inline script" do
-        subject = get_es_output({
-          'document_id' => "123",
-          'script' => 'ctx._source.counter += event["counter"]',
-          'script_lang' => 'groovy',
-          'script_type' => 'inline'
-        })
-        subject.register
-        subject.multi_receive([LogStash::Event.new("counter" => 3 )])
-        r = @es.get(:index => 'logstash-update', :type => doc_type, :id => "123", :refresh => true)
-        expect(r["_source"]["counter"]).to eq(4)
-      end
-
-      it "should increment a counter with event/doc 'count' variable with event/doc as upsert and inline script" do
-        subject = get_es_output({
-          'document_id' => "123",
-          'doc_as_upsert' => true,
-          'script' => 'if( ctx._source.containsKey("counter") ){ ctx._source.counter += event["counter"]; } else { ctx._source.counter = event["counter"]; }',
-          'script_lang' => 'groovy',
-          'script_type' => 'inline'
-        })
-        subject.register
-        subject.multi_receive([LogStash::Event.new("counter" => 3 )])
-        r = @es.get(:index => 'logstash-update', :type => doc_type, :id => "123", :refresh => true)
-        expect(r["_source"]["counter"]).to eq(4)
-      end
-
-      it "should, with new doc, set a counter with event/doc 'count' variable with event/doc as upsert and inline script" do
-        subject = get_es_output({
-          'document_id' => "456",
-          'doc_as_upsert' => true,
-          'script' => 'if( ctx._source.containsKey("counter") ){ ctx._source.counter += event["count"]; } else { ctx._source.counter = event["count"]; }',
-          'script_lang' => 'groovy',
-          'script_type' => 'inline'
-        })
-        subject.register
-        subject.multi_receive([LogStash::Event.new("counter" => 3 )])
-        r = @es.get(:index => 'logstash-update', :type => doc_type, :id => "456", :refresh => true)
-        expect(r["_source"]["counter"]).to eq(3)
-      end
-
-      it "should increment a counter with event/doc 'count' variable with indexed script" do
-        @es.put_script lang: 'groovy', id: 'indexed_update', body: { script: 'ctx._source.counter += event["count"]' }
-        subject = get_es_output({
-          'document_id' => "123",
-          'script' => 'indexed_update',
-          'script_lang' => 'groovy',
-          'script_type' => 'indexed'
-        })
-        subject.register
-        subject.multi_receive([LogStash::Event.new("count" => 4 )])
-        r = @es.get(:index => 'logstash-update', :type => doc_type, :id => "123", :refresh => true)
-        expect(r["_source"]["counter"]).to eq(5)
-      end
-    end
-
-    context "when update with upsert" do
-      it "should create new documents with provided upsert" do
-        subject = get_es_output({ 'document_id' => "456", 'upsert' => '{"message": "upsert message"}' })
-        subject.register
-        subject.multi_receive([LogStash::Event.new("message" => "sample message here")])
-        r = @es.get(:index => 'logstash-update', :type => doc_type, :id => "456", :refresh => true)
-        expect(r["_source"]["message"]).to eq('upsert message')
-      end
-
-      it "should create new documents with event/doc as upsert" do
-        subject = get_es_output({ 'document_id' => "456", 'doc_as_upsert' => true })
-        subject.register
-        subject.multi_receive([LogStash::Event.new("message" => "sample message here")])
-        r = @es.get(:index => 'logstash-update', :type => doc_type, :id => "456", :refresh => true)
-        expect(r["_source"]["message"]).to eq('sample message here')
-      end
-
-      it "should fail on documents with event/doc as upsert at external version" do
-        subject = get_es_output({ 'document_id' => "456", 'doc_as_upsert' => true, 'version' => 999, "version_type" => "external" })
-        expect { subject.register }.to raise_error(LogStash::ConfigurationError)
-      end
-    end
-
-    context "updates with scripted upsert" do
-      it "should create new documents with upsert content" do
-        subject = get_es_output({ 'document_id' => "456", 'script' => 'scripted_update', 'upsert' => '{"message": "upsert message"}', 'script_type' => 'file' })
-        subject.register
-        subject.multi_receive([LogStash::Event.new("message" => "sample message here")])
-        r = @es.get(:index => 'logstash-update', :type => doc_type, :id => "456", :refresh => true)
-        expect(r["_source"]["message"]).to eq('upsert message')
-      end
-
-      it "should create new documents with event/doc as script params" do
-        subject = get_es_output({ 'document_id' => "456", 'script' => 'scripted_upsert', 'scripted_upsert' => true, 'script_type' => 'file' })
-        subject.register
-        subject.multi_receive([LogStash::Event.new("counter" => 1)])
-        @es.indices.refresh
-        r = @es.get(:index => 'logstash-update', :type => doc_type, :id => "456", :refresh => true)
-        expect(r["_source"]["counter"]).to eq(1)
-      end
-    end
-  end
-end

data/spec/integration/outputs/templates_5x_spec.rb

@@ -1,98 +0,0 @@
-require_relative "../../../spec/es_spec_helper"
-
-if ESHelper.es_version_satisfies?(">= 5")
-  describe "index template expected behavior for 5.x", :integration => true do
-    subject! do
-      require "logstash/outputs/elasticsearch"
-      settings = {
-        "manage_template" => true,
-        "template_overwrite" => true,
-        "hosts" => "#{get_host_port()}"
-      }
-      next LogStash::Outputs::ElasticSearch.new(settings)
-    end
-
-    before :each do
-      # Delete all templates first.
-      require "elasticsearch"
-
-      # Clean ES of data before we start.
-      @es = get_client
-      @es.indices.delete_template(:name => "*")
-
-      # This can fail if there are no indexes, ignore failure.
-      @es.indices.delete(:index => "*") rescue nil
-
-      subject.register
-
-      subject.multi_receive([
-        LogStash::Event.new("message" => "sample message here"),
-        LogStash::Event.new("somemessage" => { "message" => "sample nested message here" }),
-        LogStash::Event.new("somevalue" => 100),
-        LogStash::Event.new("somevalue" => 10),
-        LogStash::Event.new("somevalue" => 1),
-        LogStash::Event.new("country" => "us"),
-        LogStash::Event.new("country" => "at"),
-        LogStash::Event.new("geoip" => { "location" => [ 0.0, 0.0 ] })
-      ])
-
-      @es.indices.refresh
-
-      # Wait or fail until everything's indexed.
-      Stud::try(20.times) do
-        r = @es.search(index: 'logstash-*')
-        expect(r).to have_hits(8)
-      end
-    end
-
-    it "permits phrase searching on string fields" do
-      results = @es.search(:q => "message:\"sample message\"")
-      expect(results).to have_hits(1)
-      expect(results["hits"]["hits"][0]["_source"]["message"]).to eq("sample message here")
-    end
-
-    it "numbers dynamically map to a numeric type and permit range queries" do
-      results = @es.search(:q => "somevalue:[5 TO 105]")
-      expect(results).to have_hits(2)
-
-      values = results["hits"]["hits"].collect { |r| r["_source"]["somevalue"] }
-      expect(values).to include(10)
-      expect(values).to include(100)
-      expect(values).to_not include(1)
-    end
-
-    it "does not create .keyword field for top-level message field" do
-      results = @es.search(:q => "message.keyword:\"sample message here\"")
-      expect(results).to have_hits(0)
-    end
-
-    it "creates .keyword field for nested message fields" do
-      results = @es.search(:q => "somemessage.message.keyword:\"sample nested message here\"")
-      expect(results).to have_hits(1)
-    end
-
-    it "creates .keyword field from any string field which is not_analyzed" do
-      results = @es.search(:q => "country.keyword:\"us\"")
-      expect(results).to have_hits(1)
-      expect(results["hits"]["hits"][0]["_source"]["country"]).to eq("us")
-
-      # partial or terms should not work.
-      results = @es.search(:q => "country.keyword:\"u\"")
-      expect(results).to have_hits(0)
-    end
-
-    it "make [geoip][location] a geo_point" do
-      expect(field_properties_from_template("logstash", "geoip")["location"]["type"]).to eq("geo_point")
-    end
-
-    it "aggregate .keyword results correctly " do
-      results = @es.search(:body => { "aggregations" => { "my_agg" => { "terms" => { "field" => "country.keyword" } } } })["aggregations"]["my_agg"]
-      terms = results["buckets"].collect { |b| b["key"] }
-
-      expect(terms).to include("us")
-
-      # 'at' is a stopword, make sure stopwords are not ignored.
-      expect(terms).to include("at")
-    end
-  end
-end