logstash-output-elasticsearch 0.1.6 → 3.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (42) hide show
  1. checksums.yaml +5 -13
  2. data/CHANGELOG.md +117 -0
  3. data/CONTRIBUTORS +32 -0
  4. data/Gemfile +4 -4
  5. data/LICENSE +1 -1
  6. data/NOTICE.TXT +5 -0
  7. data/README.md +110 -0
  8. data/lib/logstash/outputs/elasticsearch.rb +97 -425
  9. data/lib/logstash/outputs/elasticsearch/buffer.rb +124 -0
  10. data/lib/logstash/outputs/elasticsearch/common.rb +205 -0
  11. data/lib/logstash/outputs/elasticsearch/common_configs.rb +164 -0
  12. data/lib/logstash/outputs/elasticsearch/elasticsearch-template.json +36 -24
  13. data/lib/logstash/outputs/elasticsearch/http_client.rb +236 -0
  14. data/lib/logstash/outputs/elasticsearch/http_client_builder.rb +106 -0
  15. data/lib/logstash/outputs/elasticsearch/template_manager.rb +35 -0
  16. data/logstash-output-elasticsearch.gemspec +17 -15
  17. data/spec/es_spec_helper.rb +77 -0
  18. data/spec/fixtures/scripts/scripted_update.groovy +2 -0
  19. data/spec/fixtures/scripts/scripted_update_nested.groovy +2 -0
  20. data/spec/fixtures/scripts/scripted_upsert.groovy +2 -0
  21. data/spec/integration/outputs/create_spec.rb +55 -0
  22. data/spec/integration/outputs/index_spec.rb +68 -0
  23. data/spec/integration/outputs/parent_spec.rb +73 -0
  24. data/spec/integration/outputs/pipeline_spec.rb +75 -0
  25. data/spec/integration/outputs/retry_spec.rb +163 -0
  26. data/spec/integration/outputs/routing_spec.rb +65 -0
  27. data/spec/integration/outputs/secure_spec.rb +108 -0
  28. data/spec/integration/outputs/templates_spec.rb +90 -0
  29. data/spec/integration/outputs/update_spec.rb +188 -0
  30. data/spec/unit/buffer_spec.rb +118 -0
  31. data/spec/unit/http_client_builder_spec.rb +27 -0
  32. data/spec/unit/outputs/elasticsearch/http_client_spec.rb +133 -0
  33. data/spec/unit/outputs/elasticsearch_proxy_spec.rb +58 -0
  34. data/spec/unit/outputs/elasticsearch_spec.rb +227 -0
  35. data/spec/unit/outputs/elasticsearch_ssl_spec.rb +55 -0
  36. metadata +137 -51
  37. data/.gitignore +0 -4
  38. data/Rakefile +0 -6
  39. data/lib/logstash/outputs/elasticsearch/protocol.rb +0 -253
  40. data/rakelib/publish.rake +0 -9
  41. data/rakelib/vendor.rake +0 -169
  42. data/spec/outputs/elasticsearch.rb +0 -518
@@ -0,0 +1,65 @@
1
+ require_relative "../../../spec/es_spec_helper"
2
+
3
+ shared_examples "a routing indexer" do
4
+ let(:index) { 10.times.collect { rand(10).to_s }.join("") }
5
+ let(:type) { 10.times.collect { rand(10).to_s }.join("") }
6
+ let(:event_count) { 10000 + rand(500) }
7
+ let(:flush_size) { rand(200) + 1 }
8
+ let(:routing) { "not_implemented" }
9
+ let(:config) { "not_implemented" }
10
+ subject { LogStash::Outputs::ElasticSearch.new(config) }
11
+
12
+ before do
13
+ subject.register
14
+ event_count.times do
15
+ subject.receive(LogStash::Event.new("message" => "Hello World!", "type" => type))
16
+ end
17
+ end
18
+
19
+
20
+ it "ships events" do
21
+ index_url = "http://#{get_host_port()}/#{index}"
22
+
23
+ ftw = FTW::Agent.new
24
+ ftw.post!("#{index_url}/_refresh")
25
+
26
+ # Wait until all events are available.
27
+ Stud::try(10.times) do
28
+ data = ""
29
+ response = ftw.get!("#{index_url}/_count?q=*&routing=#{routing}")
30
+ response.read_body { |chunk| data << chunk }
31
+ result = LogStash::Json.load(data)
32
+ cur_count = result["count"]
33
+ insist { cur_count } == event_count
34
+ end
35
+ end
36
+ end
37
+
38
+ describe "(http protocol) index events with static routing", :integration => true do
39
+ it_behaves_like 'a routing indexer' do
40
+ let(:routing) { "test" }
41
+ let(:config) {
42
+ {
43
+ "hosts" => get_host_port,
44
+ "index" => index,
45
+ "flush_size" => flush_size,
46
+ "routing" => routing
47
+ }
48
+ }
49
+ end
50
+ end
51
+
52
+ describe "(http_protocol) index events with fieldref in routing value", :integration => true do
53
+ it_behaves_like 'a routing indexer' do
54
+ let(:routing) { "test" }
55
+ let(:config) {
56
+ {
57
+ "hosts" => get_host_port,
58
+ "index" => index,
59
+ "flush_size" => flush_size,
60
+ "routing" => "%{message}"
61
+ }
62
+ }
63
+ end
64
+ end
65
+
@@ -0,0 +1,108 @@
1
+ require_relative "../../../spec/es_spec_helper"
2
+
3
+ describe "send messages to ElasticSearch using HTTPS", :elasticsearch_secure => true do
4
+ subject do
5
+ require "logstash/outputs/elasticsearch"
6
+ settings = {
7
+ "node_name" => "logstash",
8
+ "cluster" => "elasticsearch",
9
+ "hosts" => "node01",
10
+ "user" => "user",
11
+ "password" => "changeme",
12
+ "ssl" => true,
13
+ "cacert" => "/tmp/ca/certs/cacert.pem",
14
+ # or
15
+ #"truststore" => "/tmp/ca/truststore.jks",
16
+ #"truststore_password" => "testeteste"
17
+ }
18
+ next LogStash::Outputs::ElasticSearch.new(settings)
19
+ end
20
+
21
+ before :each do
22
+ subject.register
23
+ end
24
+
25
+ it "sends events to ES" do
26
+ expect {
27
+ subject.receive(LogStash::Event.new("message" => "sample message here"))
28
+ subject.flush
29
+ }.to_not raise_error
30
+ end
31
+ end
32
+
33
+ describe "connect using HTTP Authentication", :elasticsearch_secure => true do
34
+ subject do
35
+ require "logstash/outputs/elasticsearch"
36
+ settings = {
37
+ "cluster" => "elasticsearch",
38
+ "hosts" => "node01",
39
+ "user" => "user",
40
+ "password" => "changeme",
41
+ }
42
+ next LogStash::Outputs::ElasticSearch.new(settings)
43
+ end
44
+
45
+ before :each do
46
+ subject.register
47
+ end
48
+
49
+ it "sends events to ES" do
50
+ expect {
51
+ subject.receive(LogStash::Event.new("message" => "sample message here"))
52
+ subject.flush
53
+ }.to_not raise_error
54
+ end
55
+ end
56
+
57
+ describe "send messages to ElasticSearch using HTTPS", :elasticsearch_secure => true do
58
+ subject do
59
+ require "logstash/outputs/elasticsearch"
60
+ settings = {
61
+ "node_name" => "logstash",
62
+ "cluster" => "elasticsearch",
63
+ "hosts" => "node01",
64
+ "user" => "user",
65
+ "password" => "changeme",
66
+ "ssl" => true,
67
+ "cacert" => "/tmp/ca/certs/cacert.pem",
68
+ # or
69
+ #"truststore" => "/tmp/ca/truststore.jks",
70
+ #"truststore_password" => "testeteste"
71
+ }
72
+ next LogStash::Outputs::ElasticSearch.new(settings)
73
+ end
74
+
75
+ before :each do
76
+ subject.register
77
+ end
78
+
79
+ it "sends events to ES" do
80
+ expect {
81
+ subject.receive(LogStash::Event.new("message" => "sample message here"))
82
+ subject.flush
83
+ }.to_not raise_error
84
+ end
85
+ end
86
+
87
+ describe "connect using HTTP Authentication", :elasticsearch_secure => true do
88
+ subject do
89
+ require "logstash/outputs/elasticsearch"
90
+ settings = {
91
+ "hosts" => "node01",
92
+ "user" => "user",
93
+ "password" => "changeme",
94
+ }
95
+ next LogStash::Outputs::ElasticSearch.new(settings)
96
+ end
97
+
98
+ before :each do
99
+ subject.register
100
+ end
101
+
102
+ it "sends events to ES" do
103
+ expect {
104
+ subject.receive(LogStash::Event.new("message" => "sample message here"))
105
+ subject.flush
106
+ }.to_not raise_error
107
+ end
108
+ end
@@ -0,0 +1,90 @@
1
+ require_relative "../../../spec/es_spec_helper"
2
+
3
+ describe "index template expected behavior", :integration => true do
4
+ subject! do
5
+ require "logstash/outputs/elasticsearch"
6
+ settings = {
7
+ "manage_template" => true,
8
+ "template_overwrite" => true,
9
+ "hosts" => "#{get_host_port()}"
10
+ }
11
+ next LogStash::Outputs::ElasticSearch.new(settings)
12
+ end
13
+
14
+ before :each do
15
+ # Delete all templates first.
16
+ require "elasticsearch"
17
+
18
+ # Clean ES of data before we start.
19
+ @es = get_client
20
+ @es.indices.delete_template(:name => "*")
21
+
22
+ # This can fail if there are no indexes, ignore failure.
23
+ @es.indices.delete(:index => "*") rescue nil
24
+
25
+ subject.register
26
+
27
+ subject.receive(LogStash::Event.new("message" => "sample message here"))
28
+ subject.receive(LogStash::Event.new("somevalue" => 100))
29
+ subject.receive(LogStash::Event.new("somevalue" => 10))
30
+ subject.receive(LogStash::Event.new("somevalue" => 1))
31
+ subject.receive(LogStash::Event.new("country" => "us"))
32
+ subject.receive(LogStash::Event.new("country" => "at"))
33
+ subject.receive(LogStash::Event.new("geoip" => { "location" => [ 0.0, 0.0 ] }))
34
+ subject.flush
35
+ @es.indices.refresh
36
+
37
+ # Wait or fail until everything's indexed.
38
+ Stud::try(20.times) do
39
+ r = @es.search
40
+ insist { r["hits"]["total"] } == 7
41
+ end
42
+ end
43
+
44
+ it "permits phrase searching on string fields" do
45
+ results = @es.search(:q => "message:\"sample message\"")
46
+ insist { results["hits"]["total"] } == 1
47
+ insist { results["hits"]["hits"][0]["_source"]["message"] } == "sample message here"
48
+ end
49
+
50
+ it "numbers dynamically map to a numeric type and permit range queries" do
51
+ results = @es.search(:q => "somevalue:[5 TO 105]")
52
+ insist { results["hits"]["total"] } == 2
53
+
54
+ values = results["hits"]["hits"].collect { |r| r["_source"]["somevalue"] }
55
+ insist { values }.include?(10)
56
+ insist { values }.include?(100)
57
+ reject { values }.include?(1)
58
+ end
59
+
60
+ it "does not create .raw field for the message field" do
61
+ results = @es.search(:q => "message.raw:\"sample message here\"")
62
+ insist { results["hits"]["total"] } == 0
63
+ end
64
+
65
+ it "creates .raw field from any string field which is not_analyzed" do
66
+ results = @es.search(:q => "country.raw:\"us\"")
67
+ insist { results["hits"]["total"] } == 1
68
+ insist { results["hits"]["hits"][0]["_source"]["country"] } == "us"
69
+
70
+ # partial or terms should not work.
71
+ results = @es.search(:q => "country.raw:\"u\"")
72
+ insist { results["hits"]["total"] } == 0
73
+ end
74
+
75
+ it "make [geoip][location] a geo_point" do
76
+ results = @es.search(:body => { "query" => { "bool" => { "must" => { "match_all" => {} }, "filter" => { "geo_distance" => { "distance" => "1000km", "geoip.location" => { "lat" => 0.5, "lon" => 0.5 } } } } } })
77
+ insist { results["hits"]["total"] } == 1
78
+ insist { results["hits"]["hits"][0]["_source"]["geoip"]["location"] } == [ 0.0, 0.0 ]
79
+ end
80
+
81
+ it "aggregate .raw results correctly " do
82
+ results = @es.search(:body => { "aggregations" => { "my_agg" => { "terms" => { "field" => "country.raw" } } } })["aggregations"]["my_agg"]
83
+ terms = results["buckets"].collect { |b| b["key"] }
84
+
85
+ insist { terms }.include?("us")
86
+
87
+ # 'at' is a stopword, make sure stopwords are not ignored.
88
+ insist { terms }.include?("at")
89
+ end
90
+ end
@@ -0,0 +1,188 @@
1
+ require_relative "../../../spec/es_spec_helper"
2
+
3
+ describe "Update actions", :integration => true do
4
+ require "logstash/outputs/elasticsearch"
5
+ require "elasticsearch"
6
+
7
+ def get_es_output( options={} )
8
+ settings = {
9
+ "manage_template" => true,
10
+ "index" => "logstash-update",
11
+ "template_overwrite" => true,
12
+ "hosts" => get_host_port(),
13
+ "action" => "update"
14
+ }
15
+ LogStash::Outputs::ElasticSearch.new(settings.merge!(options))
16
+ end
17
+
18
+ before :each do
19
+ @es = get_client
20
+ # Delete all templates first.
21
+ # Clean ES of data before we start.
22
+ @es.indices.delete_template(:name => "*")
23
+ # This can fail if there are no indexes, ignore failure.
24
+ @es.indices.delete(:index => "*") rescue nil
25
+ @es.index(
26
+ :index => 'logstash-update',
27
+ :type => 'logs',
28
+ :id => "123",
29
+ :body => { :message => 'Test', :counter => 1 }
30
+ )
31
+ @es.indices.refresh
32
+ end
33
+
34
+ it "should fail without a document_id" do
35
+ subject = get_es_output
36
+ expect { subject.register }.to raise_error(LogStash::ConfigurationError)
37
+ end
38
+
39
+ context "when update only" do
40
+ it "should not create new document" do
41
+ subject = get_es_output({ 'document_id' => "456" } )
42
+ subject.register
43
+ subject.receive(LogStash::Event.new("message" => "sample message here"))
44
+ subject.flush
45
+ expect {@es.get(:index => 'logstash-update', :type => 'logs', :id => "456", :refresh => true)}.to raise_error(Elasticsearch::Transport::Transport::Errors::NotFound)
46
+ end
47
+
48
+ it "should update existing document" do
49
+ subject = get_es_output({ 'document_id' => "123" })
50
+ subject.register
51
+ subject.receive(LogStash::Event.new("message" => "updated message here"))
52
+ subject.flush
53
+ r = @es.get(:index => 'logstash-update', :type => 'logs', :id => "123", :refresh => true)
54
+ insist { r["_source"]["message"] } == 'updated message here'
55
+ end
56
+
57
+ # The es ruby client treats the data field differently. Make sure this doesn't
58
+ # raise an exception
59
+ it "should update an existing document that has a 'data' field" do
60
+ subject = get_es_output({ 'document_id' => "123" })
61
+ subject.register
62
+ subject.receive(LogStash::Event.new("data" => "updated message here", "message" => "foo"))
63
+ subject.flush
64
+ r = @es.get(:index => 'logstash-update', :type => 'logs', :id => "123", :refresh => true)
65
+ insist { r["_source"]["data"] } == 'updated message here'
66
+ insist { r["_source"]["message"] } == 'foo'
67
+ end
68
+ end
69
+
70
+ context "when using script" do
71
+ it "should increment a counter with event/doc 'count' variable" do
72
+ subject = get_es_output({ 'document_id' => "123", 'script' => 'scripted_update', 'script_type' => 'file' })
73
+ subject.register
74
+ subject.receive(LogStash::Event.new("count" => 2))
75
+ subject.flush
76
+ r = @es.get(:index => 'logstash-update', :type => 'logs', :id => "123", :refresh => true)
77
+ insist { r["_source"]["counter"] } == 3
78
+ end
79
+
80
+ it "should increment a counter with event/doc '[data][count]' nested variable" do
81
+ subject = get_es_output({ 'document_id' => "123", 'script' => 'scripted_update_nested', 'script_type' => 'file' })
82
+ subject.register
83
+ subject.receive(LogStash::Event.new("data" => { "count" => 3 }))
84
+ subject.flush
85
+ r = @es.get(:index => 'logstash-update', :type => 'logs', :id => "123", :refresh => true)
86
+ insist { r["_source"]["counter"] } == 4
87
+ end
88
+
89
+ it "should increment a counter with event/doc 'count' variable with inline script" do
90
+ subject = get_es_output({
91
+ 'document_id' => "123",
92
+ 'script' => 'ctx._source.counter += event["counter"]',
93
+ 'script_lang' => 'groovy',
94
+ 'script_type' => 'inline'
95
+ })
96
+ subject.register
97
+ subject.receive(LogStash::Event.new("counter" => 3 ))
98
+ subject.flush
99
+ r = @es.get(:index => 'logstash-update', :type => 'logs', :id => "123", :refresh => true)
100
+ insist { r["_source"]["counter"] } == 4
101
+ end
102
+
103
+ it "should increment a counter with event/doc 'count' variable with event/doc as upsert and inline script" do
104
+ subject = get_es_output({
105
+ 'document_id' => "123",
106
+ 'doc_as_upsert' => true,
107
+ 'script' => 'if( ctx._source.containsKey("counter") ){ ctx._source.counter += event["counter"]; } else { ctx._source.counter = event["counter"]; }',
108
+ 'script_lang' => 'groovy',
109
+ 'script_type' => 'inline'
110
+ })
111
+ subject.register
112
+ subject.receive(LogStash::Event.new("counter" => 3 ))
113
+ subject.flush
114
+ r = @es.get(:index => 'logstash-update', :type => 'logs', :id => "123", :refresh => true)
115
+ insist { r["_source"]["counter"] } == 4
116
+ end
117
+
118
+ it "should, with new doc, set a counter with event/doc 'count' variable with event/doc as upsert and inline script" do
119
+ subject = get_es_output({
120
+ 'document_id' => "456",
121
+ 'doc_as_upsert' => true,
122
+ 'script' => 'if( ctx._source.containsKey("counter") ){ ctx._source.counter += event["count"]; } else { ctx._source.counter = event["count"]; }',
123
+ 'script_lang' => 'groovy',
124
+ 'script_type' => 'inline'
125
+ })
126
+ subject.register
127
+ subject.receive(LogStash::Event.new("counter" => 3 ))
128
+ subject.flush
129
+ r = @es.get(:index => 'logstash-update', :type => 'logs', :id => "456", :refresh => true)
130
+ insist { r["_source"]["counter"] } == 3
131
+ end
132
+
133
+ it "should increment a counter with event/doc 'count' variable with indexed script" do
134
+ @es.put_script lang: 'groovy', id: 'indexed_update', body: { script: 'ctx._source.counter += event["count"]' }
135
+ subject = get_es_output({
136
+ 'document_id' => "123",
137
+ 'script' => 'indexed_update',
138
+ 'script_lang' => 'groovy',
139
+ 'script_type' => 'indexed'
140
+ })
141
+ subject.register
142
+ subject.receive(LogStash::Event.new("count" => 4 ))
143
+ subject.flush
144
+ r = @es.get(:index => 'logstash-update', :type => 'logs', :id => "123", :refresh => true)
145
+ insist { r["_source"]["counter"] } == 5
146
+ end
147
+ end
148
+
149
+ context "when update with upsert" do
150
+ it "should create new documents with provided upsert" do
151
+ subject = get_es_output({ 'document_id' => "456", 'upsert' => '{"message": "upsert message"}' })
152
+ subject.register
153
+ subject.receive(LogStash::Event.new("message" => "sample message here"))
154
+ subject.flush
155
+ r = @es.get(:index => 'logstash-update', :type => 'logs', :id => "456", :refresh => true)
156
+ insist { r["_source"]["message"] } == 'upsert message'
157
+ end
158
+
159
+ it "should create new documents with event/doc as upsert" do
160
+ subject = get_es_output({ 'document_id' => "456", 'doc_as_upsert' => true })
161
+ subject.register
162
+ subject.receive(LogStash::Event.new("message" => "sample message here"))
163
+ subject.flush
164
+ r = @es.get(:index => 'logstash-update', :type => 'logs', :id => "456", :refresh => true)
165
+ insist { r["_source"]["message"] } == 'sample message here'
166
+ end
167
+
168
+ context "when using script" do
169
+ it "should create new documents with upsert content" do
170
+ subject = get_es_output({ 'document_id' => "456", 'script' => 'scripted_update', 'upsert' => '{"message": "upsert message"}', 'script_type' => 'file' })
171
+ subject.register
172
+ subject.receive(LogStash::Event.new("message" => "sample message here"))
173
+ subject.flush
174
+ r = @es.get(:index => 'logstash-update', :type => 'logs', :id => "456", :refresh => true)
175
+ insist { r["_source"]["message"] } == 'upsert message'
176
+ end
177
+
178
+ it "should create new documents with event/doc as script params" do
179
+ subject = get_es_output({ 'document_id' => "456", 'script' => 'scripted_upsert', 'scripted_upsert' => true, 'script_type' => 'file' })
180
+ subject.register
181
+ subject.receive(LogStash::Event.new("counter" => 1))
182
+ subject.flush
183
+ r = @es.get(:index => 'logstash-update', :type => 'logs', :id => "456", :refresh => true)
184
+ insist { r["_source"]["counter"] } == 1
185
+ end
186
+ end
187
+ end
188
+ end