logstash-output-elastic_app_search 1.1.1 → 1.2.0
Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 074b1a0e77284e01d820135619726f4cdcc5e0626bae4ba694d1c08870184f46
|
4
|
+
data.tar.gz: 5fac5d1794454fd80ec91942a15a34497c709a74a64e6b3e19eebb184f6c4746
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 8244e5bc5378bdb4b05ff48ee7da23f549e6e62f47b828b2c8f384633e635cf3fb3a5a16dba6b5b7a9028ab2c1b431a3227c45e181584b386ba547fd6384e839
|
7
|
+
data.tar.gz: 4e00d97b9d9d360dfa3bfb747797b35efc888e868c5663154b296d4a8716fbd96c42aae0b1742458f56a4bfdd2d4c75b9affa57fdd7961f0bd993241c21fa903
|
data/CHANGELOG.md
CHANGED
@@ -1,3 +1,6 @@
|
|
1
|
+
## 1.2.0
|
2
|
+
- Changed evaluation of `engine` option to use event's sprintf format, [#25](https://github.com/logstash-plugins/logstash-output-elastic_app_search/pull/25)
|
3
|
+
|
1
4
|
## 1.1.1
|
2
5
|
- Added missed dependency (elastic-app-search) to the gemspec, fixes issue [#17](https://github.com/logstash-plugins/logstash-output-elastic_app_search/issues/17)
|
3
6
|
|
@@ -13,6 +13,8 @@ class LogStash::Outputs::ElasticAppSearch < LogStash::Outputs::Base
|
|
13
13
|
config :document_id, :validate => :string
|
14
14
|
config :path, :validate => :string, :default => "/api/as/v1/"
|
15
15
|
|
16
|
+
ENGINE_WITH_SPRINTF_REGEX = /^.*%\{.+\}.*$/
|
17
|
+
|
16
18
|
public
|
17
19
|
def register
|
18
20
|
if @host.nil? && @url.nil?
|
@@ -26,7 +28,7 @@ class LogStash::Outputs::ElasticAppSearch < LogStash::Outputs::Base
|
|
26
28
|
elsif @url
|
27
29
|
@client = Elastic::AppSearch::Client.new(:api_endpoint => @url + @path, :api_key => @api_key.value)
|
28
30
|
end
|
29
|
-
check_connection!
|
31
|
+
check_connection! unless @engine =~ ENGINE_WITH_SPRINTF_REGEX
|
30
32
|
rescue => e
|
31
33
|
if e.message =~ /401/
|
32
34
|
raise ::LogStash::ConfigurationError.new("Failed to connect to App Search. Error: 401. Please check your credentials")
|
@@ -51,7 +53,8 @@ class LogStash::Outputs::ElasticAppSearch < LogStash::Outputs::Base
|
|
51
53
|
|
52
54
|
private
|
53
55
|
def format_batch(events)
|
54
|
-
|
56
|
+
docs_for_engine = {}
|
57
|
+
events.each do |event|
|
55
58
|
doc = event.to_hash
|
56
59
|
# we need to remove default fields that start with "@"
|
57
60
|
# since Elastic App Search doesn't accept them
|
@@ -64,17 +67,33 @@ class LogStash::Outputs::ElasticAppSearch < LogStash::Outputs::Base
|
|
64
67
|
doc["id"] = event.sprintf(@document_id)
|
65
68
|
end
|
66
69
|
doc.delete("@version")
|
67
|
-
|
70
|
+
resolved_engine = event.sprintf(@engine)
|
71
|
+
unless docs_for_engine[resolved_engine]
|
72
|
+
if @logger.debug?
|
73
|
+
@logger.debug("Creating new engine segment in batch to send", :resolved_engine => resolved_engine)
|
74
|
+
end
|
75
|
+
docs_for_engine[resolved_engine] = []
|
76
|
+
end
|
77
|
+
docs_for_engine[resolved_engine] << doc
|
68
78
|
end
|
79
|
+
docs_for_engine
|
69
80
|
end
|
70
81
|
|
71
|
-
def index(
|
72
|
-
|
73
|
-
|
74
|
-
|
75
|
-
|
76
|
-
|
77
|
-
|
82
|
+
def index(batch)
|
83
|
+
batch.each do |resolved_engine, documents|
|
84
|
+
begin
|
85
|
+
if resolved_engine =~ ENGINE_WITH_SPRINTF_REGEX || resolved_engine =~ /^\s*$/
|
86
|
+
raise "Cannot resolve engine field name #{@engine} from event"
|
87
|
+
end
|
88
|
+
response = @client.index_documents(resolved_engine, documents)
|
89
|
+
report(documents, response)
|
90
|
+
rescue => e
|
91
|
+
@logger.error("Failed to execute index operation. Retrying..", :exception => e.class, :reason => e.message,
|
92
|
+
:resolved_engine => resolved_engine)
|
93
|
+
sleep(1)
|
94
|
+
retry
|
95
|
+
end
|
96
|
+
end
|
78
97
|
end
|
79
98
|
|
80
99
|
def report(documents, response)
|
@@ -1,6 +1,6 @@
|
|
1
1
|
Gem::Specification.new do |s|
|
2
2
|
s.name = 'logstash-output-elastic_app_search'
|
3
|
-
s.version = '1.
|
3
|
+
s.version = '1.2.0'
|
4
4
|
s.licenses = ['Apache-2.0']
|
5
5
|
s.summary = 'Index data to Elastic App Search'
|
6
6
|
s.description = 'This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program'
|
@@ -48,6 +48,22 @@ describe "indexing against running AppSearch", :integration => true do
|
|
48
48
|
end
|
49
49
|
end
|
50
50
|
|
51
|
+
describe "register" do
|
52
|
+
let(:config) do
|
53
|
+
{
|
54
|
+
"api_key" => ENV['APPSEARCH_PRIVATE_KEY'],
|
55
|
+
"engine" => "%{engine_name_field}",
|
56
|
+
"url" => "http://appsearch:3002"
|
57
|
+
}
|
58
|
+
end
|
59
|
+
|
60
|
+
context "when engine is defined in sprintf format" do
|
61
|
+
it "does not raise an error" do
|
62
|
+
expect { subject.register }.to_not raise_error
|
63
|
+
end
|
64
|
+
end
|
65
|
+
end
|
66
|
+
|
51
67
|
describe "indexing" do
|
52
68
|
|
53
69
|
before do
|
@@ -69,6 +85,31 @@ describe "indexing against running AppSearch", :integration => true do
|
|
69
85
|
end
|
70
86
|
expect(results.first.dig("message", "raw")).to eq "an event to index"
|
71
87
|
end
|
88
|
+
|
89
|
+
context "using sprintf-ed engine" do
|
90
|
+
let(:config) do
|
91
|
+
{
|
92
|
+
"api_key" => ENV['APPSEARCH_PRIVATE_KEY'],
|
93
|
+
"engine" => "%{engine_name_field}",
|
94
|
+
"url" => "http://appsearch:3002"
|
95
|
+
}
|
96
|
+
end
|
97
|
+
|
98
|
+
let(:event) { LogStash::Event.new("message" => "an event to index", "engine_name_field" => engine_name) }
|
99
|
+
|
100
|
+
it "should be indexed" do
|
101
|
+
app_search_output.multi_receive([event])
|
102
|
+
|
103
|
+
results = Stud.try(20.times, RSpec::Expectations::ExpectationNotMetError) do
|
104
|
+
attempt_response = execute_search_call(engine_name)
|
105
|
+
expect(attempt_response.status).to eq(200)
|
106
|
+
parsed_resp = JSON.parse(attempt_response.body)
|
107
|
+
expect(parsed_resp.dig("meta", "page", "total_pages")).to eq(1)
|
108
|
+
parsed_resp["results"]
|
109
|
+
end
|
110
|
+
expect(results.first.dig("message", "raw")).to eq "an event to index"
|
111
|
+
end
|
112
|
+
end
|
72
113
|
end
|
73
114
|
|
74
115
|
private
|
@@ -80,24 +121,60 @@ describe "indexing against running AppSearch", :integration => true do
|
|
80
121
|
end
|
81
122
|
|
82
123
|
describe "multiple events" do
|
83
|
-
|
124
|
+
context "single static engine" do
|
125
|
+
let(:events) { generate_events(200) } #2 times the slice size used to batch
|
84
126
|
|
85
|
-
|
86
|
-
|
87
|
-
|
88
|
-
|
89
|
-
|
90
|
-
|
91
|
-
|
92
|
-
|
127
|
+
it "all should be indexed" do
|
128
|
+
app_search_output.multi_receive(events)
|
129
|
+
|
130
|
+
expect_indexed(engine_name, 200)
|
131
|
+
end
|
132
|
+
end
|
133
|
+
|
134
|
+
context "multiple sprintf engines" do
|
135
|
+
let(:config) do
|
136
|
+
{
|
137
|
+
"api_key" => ENV['APPSEARCH_PRIVATE_KEY'],
|
138
|
+
"engine" => "%{engine_name_field}",
|
139
|
+
"url" => "http://appsearch:3002"
|
140
|
+
}
|
141
|
+
end
|
142
|
+
|
143
|
+
it "all should be indexed" do
|
144
|
+
create_engine('testengin1', "http://appsearch:3002", ENV['APPSEARCH_PRIVATE_KEY'])
|
145
|
+
create_engine('testengin2', "http://appsearch:3002", ENV['APPSEARCH_PRIVATE_KEY'])
|
146
|
+
events = generate_events(100, 'testengin1')
|
147
|
+
events += generate_events(100, 'testengin2')
|
148
|
+
events.shuffle!
|
149
|
+
|
150
|
+
app_search_output.multi_receive(events)
|
151
|
+
|
152
|
+
expect_indexed('testengin1', 100)
|
153
|
+
expect_indexed('testengin2', 100)
|
93
154
|
end
|
94
|
-
expect(results.first.dig("message", "raw")).to start_with("an event to index")
|
95
155
|
end
|
96
156
|
end
|
97
157
|
|
98
158
|
private
|
99
|
-
def
|
100
|
-
|
159
|
+
def expect_indexed(engine_name, expected_docs_count)
|
160
|
+
results = Stud.try(20.times, RSpec::Expectations::ExpectationNotMetError) do
|
161
|
+
attempt_response = execute_search_call(engine_name)
|
162
|
+
expect(attempt_response.status).to eq(200)
|
163
|
+
parsed_resp = JSON.parse(attempt_response.body)
|
164
|
+
expect(parsed_resp.dig("meta", "page", "total_results")).to eq(expected_docs_count)
|
165
|
+
parsed_resp["results"]
|
166
|
+
end
|
167
|
+
expect(results.first.dig("message", "raw")).to start_with("an event to index")
|
168
|
+
end
|
169
|
+
|
170
|
+
def generate_events(num_events, engine_name = nil)
|
171
|
+
(1..num_events).map do |i|
|
172
|
+
if engine_name
|
173
|
+
LogStash::Event.new("message" => "an event to index #{i}", "engine_name_field" => engine_name)
|
174
|
+
else
|
175
|
+
LogStash::Event.new("message" => "an event to index #{i}")
|
176
|
+
end
|
177
|
+
end
|
101
178
|
end
|
102
179
|
end
|
103
180
|
end
|
@@ -39,5 +39,12 @@ describe LogStash::Outputs::ElasticAppSearch do
|
|
39
39
|
expect { subject.register }.to raise_error(LogStash::ConfigurationError)
|
40
40
|
end
|
41
41
|
end
|
42
|
+
context "when engine is in sprintf format" do
|
43
|
+
let(:config) { { "host" => host, "api_key" => api_key, "engine" => "%{type}" } }
|
44
|
+
it "connection is not checked" do
|
45
|
+
expect { subject.register }.to_not raise_error
|
46
|
+
expect(subject).not_to receive(:check_connection!)
|
47
|
+
end
|
48
|
+
end
|
42
49
|
end
|
43
50
|
end
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: logstash-output-elastic_app_search
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.
|
4
|
+
version: 1.2.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Joao Duarte
|
@@ -9,7 +9,7 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date:
|
12
|
+
date: 2021-05-04 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
requirement: !ruby/object:Gem::Requirement
|