logstash-output-elastic_app_search 1.1.1 → 1.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 4db812e586bb58a374dbf078f9d2fa42e43eae9a8f53f08dee0913e9690be733
-  data.tar.gz: da43265b1a72eeaaa943291a9bc18f7aca3e87403461c01ffbfcc13152e4359d
+  metadata.gz: 074b1a0e77284e01d820135619726f4cdcc5e0626bae4ba694d1c08870184f46
+  data.tar.gz: 5fac5d1794454fd80ec91942a15a34497c709a74a64e6b3e19eebb184f6c4746
 SHA512:
-  metadata.gz: a01665272fe1993eefb689bc088fe32fdbaebbb1fe46dbff6f9cb8e90220dae1008612bfbefa92a2a3ea033bf86975d10d976feff2d0247affa4330bbbbe04d5
-  data.tar.gz: a5f9601d14d30b373ed918652afcd822d6bd17bd0cb62789c591ed2b443a55631cc593d0fcc6faef565466f8177056afcbda803229361fdec2f944384e02b345
+  metadata.gz: 8244e5bc5378bdb4b05ff48ee7da23f549e6e62f47b828b2c8f384633e635cf3fb3a5a16dba6b5b7a9028ab2c1b431a3227c45e181584b386ba547fd6384e839
+  data.tar.gz: 4e00d97b9d9d360dfa3bfb747797b35efc888e868c5663154b296d4a8716fbd96c42aae0b1742458f56a4bfdd2d4c75b9affa57fdd7961f0bd993241c21fa903

data/CHANGELOG.md

@@ -1,3 +1,6 @@
+## 1.2.0
+  - Changed evaluation of `engine` option to use event's sprintf format, [#25](https://github.com/logstash-plugins/logstash-output-elastic_app_search/pull/25)
+
 ## 1.1.1
   - Added missed dependency (elastic-app-search) to the gemspec, fixes issue [#17](https://github.com/logstash-plugins/logstash-output-elastic_app_search/issues/17)
 

data/lib/logstash/outputs/elastic_app_search.rb

@@ -13,6 +13,8 @@ class LogStash::Outputs::ElasticAppSearch < LogStash::Outputs::Base
   config :document_id, :validate => :string
   config :path, :validate => :string, :default => "/api/as/v1/"
 
+  ENGINE_WITH_SPRINTF_REGEX = /^.*%\{.+\}.*$/
+
   public
   def register
     if @host.nil? && @url.nil?
@@ -26,7 +28,7 @@ class LogStash::Outputs::ElasticAppSearch < LogStash::Outputs::Base
     elsif @url
       @client = Elastic::AppSearch::Client.new(:api_endpoint => @url + @path, :api_key => @api_key.value)
     end
-    check_connection!
+    check_connection! unless @engine =~ ENGINE_WITH_SPRINTF_REGEX
   rescue => e
     if e.message =~ /401/
       raise ::LogStash::ConfigurationError.new("Failed to connect to App Search. Error: 401. Please check your credentials")
@@ -51,7 +53,8 @@ class LogStash::Outputs::ElasticAppSearch < LogStash::Outputs::Base
 
   private
   def format_batch(events)
-    events.map do |event|
+    docs_for_engine = {}
+    events.each do |event|
       doc = event.to_hash
       # we need to remove default fields that start with "@"
       # since Elastic App Search doesn't accept them
@@ -64,17 +67,33 @@ class LogStash::Outputs::ElasticAppSearch < LogStash::Outputs::Base
         doc["id"] = event.sprintf(@document_id)
       end
       doc.delete("@version")
-      doc
+      resolved_engine = event.sprintf(@engine)
+      unless docs_for_engine[resolved_engine]
+        if @logger.debug?
+          @logger.debug("Creating new engine segment in batch to send", :resolved_engine => resolved_engine)
+        end
+        docs_for_engine[resolved_engine] = []
+      end
+      docs_for_engine[resolved_engine] << doc
     end
+    docs_for_engine
   end
 
-  def index(documents)
-    response = @client.index_documents(@engine, documents)
-    report(documents, response)
-  rescue => e
-    @logger.error("Failed to execute index operation. Retrying..", :exception => e.class, :reason => e.message)
-    sleep(1)
-    retry
+  def index(batch)
+    batch.each do |resolved_engine, documents|
+      begin
+        if resolved_engine =~ ENGINE_WITH_SPRINTF_REGEX || resolved_engine =~ /^\s*$/
+          raise "Cannot resolve engine field name #{@engine} from event"
+        end
+        response = @client.index_documents(resolved_engine, documents)
+        report(documents, response)
+      rescue => e
+        @logger.error("Failed to execute index operation. Retrying..", :exception => e.class, :reason => e.message,
+                      :resolved_engine => resolved_engine)
+        sleep(1)
+        retry
+      end
+    end
   end
 
   def report(documents, response)

data/logstash-output-elastic_app_search.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name          = 'logstash-output-elastic_app_search'
-  s.version       = '1.1.1'
+  s.version       = '1.2.0'
   s.licenses      = ['Apache-2.0']
   s.summary       = 'Index data to Elastic App Search'
   s.description   = 'This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program'

data/spec/integration/outputs/index_spec.rb

@@ -48,6 +48,22 @@ describe "indexing against running AppSearch", :integration => true do
     end
   end
 
+  describe "register" do
+    let(:config) do
+      {
+        "api_key" => ENV['APPSEARCH_PRIVATE_KEY'],
+        "engine" => "%{engine_name_field}",
+        "url" => "http://appsearch:3002"
+      }
+    end
+
+    context "when engine is defined in sprintf format" do
+      it "does not raise an error" do
+        expect { subject.register }.to_not raise_error
+      end
+    end
+  end
+
   describe "indexing" do
 
     before do
@@ -69,6 +85,31 @@ describe "indexing against running AppSearch", :integration => true do
         end
         expect(results.first.dig("message", "raw")).to eq "an event to index"
       end
+
+      context "using sprintf-ed engine" do
+        let(:config) do
+          {
+            "api_key" => ENV['APPSEARCH_PRIVATE_KEY'],
+            "engine" => "%{engine_name_field}",
+            "url" => "http://appsearch:3002"
+          }
+        end
+
+        let(:event) { LogStash::Event.new("message" => "an event to index", "engine_name_field" => engine_name) }
+
+        it "should be indexed" do
+          app_search_output.multi_receive([event])
+
+          results = Stud.try(20.times, RSpec::Expectations::ExpectationNotMetError) do
+            attempt_response = execute_search_call(engine_name)
+            expect(attempt_response.status).to eq(200)
+            parsed_resp = JSON.parse(attempt_response.body)
+            expect(parsed_resp.dig("meta", "page", "total_pages")).to eq(1)
+            parsed_resp["results"]
+          end
+          expect(results.first.dig("message", "raw")).to eq "an event to index"
+        end
+      end
     end
 
     private
@@ -80,24 +121,60 @@ describe "indexing against running AppSearch", :integration => true do
     end
 
     describe "multiple events" do
-      let(:events) { generate_events(200) } #2 times the slice size used to batch
+      context "single static engine" do
+        let(:events) { generate_events(200) } #2 times the slice size used to batch
 
-      it "all should be indexed" do
-        app_search_output.multi_receive(events)
-        results = Stud.try(20.times, RSpec::Expectations::ExpectationNotMetError) do
-          attempt_response = execute_search_call(engine_name)
-          expect(attempt_response.status).to eq(200)
-          parsed_resp = JSON.parse(attempt_response.body)
-          expect(parsed_resp.dig("meta", "page", "total_results")).to eq(200)
-          parsed_resp["results"]
+        it "all should be indexed" do
+          app_search_output.multi_receive(events)
+
+          expect_indexed(engine_name, 200)
+        end
+      end
+
+      context "multiple sprintf engines" do
+        let(:config) do
+          {
+            "api_key" => ENV['APPSEARCH_PRIVATE_KEY'],
+            "engine" => "%{engine_name_field}",
+            "url" => "http://appsearch:3002"
+          }
+        end
+
+        it "all should be indexed" do
+         create_engine('testengin1', "http://appsearch:3002", ENV['APPSEARCH_PRIVATE_KEY'])
+         create_engine('testengin2', "http://appsearch:3002", ENV['APPSEARCH_PRIVATE_KEY'])
+         events = generate_events(100, 'testengin1')
+         events += generate_events(100, 'testengin2')
+         events.shuffle!
+
+         app_search_output.multi_receive(events)
+
+         expect_indexed('testengin1', 100)
+         expect_indexed('testengin2', 100)
         end
-        expect(results.first.dig("message", "raw")).to start_with("an event to index")
       end
     end
 
     private
-    def generate_events(num_events)
-      (1..num_events).map { |i| LogStash::Event.new("message" => "an event to index #{i}")}
+    def expect_indexed(engine_name, expected_docs_count)
+      results = Stud.try(20.times, RSpec::Expectations::ExpectationNotMetError) do
+        attempt_response = execute_search_call(engine_name)
+        expect(attempt_response.status).to eq(200)
+        parsed_resp = JSON.parse(attempt_response.body)
+        expect(parsed_resp.dig("meta", "page", "total_results")).to eq(expected_docs_count)
+        parsed_resp["results"]
+      end
+      expect(results.first.dig("message", "raw")).to start_with("an event to index")
+    end
+
+    def generate_events(num_events, engine_name = nil)
+      (1..num_events).map do |i|
+        if engine_name
+          LogStash::Event.new("message" => "an event to index #{i}", "engine_name_field" => engine_name)
+        else
+          LogStash::Event.new("message" => "an event to index #{i}")
+        end
+      end
     end
   end
 end

data/spec/outputs/appsearch_spec.rb

@@ -39,5 +39,12 @@ describe LogStash::Outputs::ElasticAppSearch do
         expect { subject.register }.to raise_error(LogStash::ConfigurationError)
       end
     end
+    context "when engine is in sprintf format" do
+      let(:config) { { "host" => host, "api_key" => api_key, "engine" => "%{type}" } }
+      it "connection is not checked" do
+        expect { subject.register }.to_not raise_error
+        expect(subject).not_to receive(:check_connection!)
+      end
+    end
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: logstash-output-elastic_app_search
 version: !ruby/object:Gem::Version
-  version: 1.1.1
+  version: 1.2.0
 platform: ruby
 authors:
 - Joao Duarte
@@ -9,7 +9,7 @@ authors:
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2020-09-16 00:00:00.000000000 Z
+date: 2021-05-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement